Add trip ID cleanup regex + early route/stop ID cleanup

src/main/java/org/mtransit/parser/DefaultAgencyTools.java

@@ -353,7 +353,7 @@ public String cleanRouteOriginalId(@NotNull String gRouteId) {
 	public long getRouteId(@NotNull GRoute gRoute) {
 		try {
 			//noinspection DiscouragedApi
-			final String routeIdS = useRouteShortNameForRouteId() ? cleanRouteShortName(getRouteShortName(gRoute)) : cleanRouteOriginalId(gRoute.getRouteId());
+			final String routeIdS = useRouteShortNameForRouteId() ? cleanRouteShortName(getRouteShortName(gRoute)) : gRoute.getRouteId();
 			if (defaultRouteIdEnabled()
 					&& !CharUtils.isDigitsOnly(routeIdS)) {
 				return MRouteSNToIDConverter.convert(
@@ -601,6 +601,31 @@ public String cleanTripHeadsign(@NotNull String tripHeadsign) {
 		return tripHeadsign;
 	}
 
+	@Nullable
+	@Override
+	public String getTripIdCleanupRegex() {
+		return Configs.getRouteConfig().getTripIdCleanupRegex();
+	}
+
+	@Nullable
+	private Pattern tripIdCleanupPattern = null;
+
+	private boolean tripIdCleanupPatternSet = false;
+
+	@Override
+	public @Nullable Pattern getTripIdCleanupPattern() {
+		if (this.tripIdCleanupPattern == null && !tripIdCleanupPatternSet) {
+			this.tripIdCleanupPattern = GTFSCommons.makeIdCleanupPattern(getTripIdCleanupRegex());
+			this.tripIdCleanupPatternSet = true;
+		}
+		return this.tripIdCleanupPattern;
+	}
+
+	@Override
+	public @NotNull String cleanTripOriginalId(@NotNull String gTripId) {
+		return GTFSCommons.cleanOriginalId(gTripId, getTripIdCleanupPattern());
+	}
+
 	@Override
 	public boolean directionSplitterEnabled(long routeId) {
 		return false; // OPT-IN feature
@@ -926,7 +951,7 @@ public String getStopOriginalId(@NotNull GStop gStop) {
 	public int getStopId(@NotNull GStop gStop) {
 		try {
 			//noinspection DiscouragedApi
-			final String stopIdS = useStopCodeForStopId() ? getStopCode(gStop) : cleanStopOriginalId(gStop.getStopId());
+			final String stopIdS = useStopCodeForStopId() ? getStopCode(gStop) : gStop.getStopId();
 			//noinspection DiscouragedApi
 			return Integer.parseInt(stopIdS);
 		} catch (Exception e) {

src/main/java/org/mtransit/parser/config/gtfs/data/RouteConfig.kt

@@ -35,6 +35,8 @@ data class RouteConfig(
     @SerialName("trip_headsign_remove_via")
     val tripHeadsignRemoveVia: Boolean = false, // OPT-IN feature
     // DIRECTION
+    @SerialName("trip_id_cleanup_regex")
+    val tripIdCleanupRegex: String? = null, // optional
     @SerialName("direction_headsign_cleaners")
     val directionHeadsignCleaners: List<Cleaner> = emptyList(),
     @SerialName("direction_finder_enabled")

src/main/java/org/mtransit/parser/db/GTFSDataBase.kt

@@ -156,9 +156,9 @@ object GTFSDataBase {
 
     @JvmOverloads
     @JvmStatic
-    fun insertRoute(route: Route, preparedStatement: PreparedStatement? = null) {
+    fun insertRoute(route: Route, allowUpdate: Boolean = false, preparedStatement: PreparedStatement? = null) {
         connection.createStatement().use { statement ->
-            RouteSQL.insertIntoMainTable(route, statement, preparedStatement)
+            RouteSQL.insertIntoMainTable(route, statement, preparedStatement, allowUpdate)
         }
     }
 

src/main/java/org/mtransit/parser/gtfs/GAgencyTools.java

@@ -159,6 +159,14 @@ public interface GAgencyTools {
 	@NotNull
 	String cleanTripHeadsign(@NotNull String tripHeadsign);
 
+	@Nullable
+	String getTripIdCleanupRegex();
+
+	@Nullable
+	Pattern getTripIdCleanupPattern();
+
+	@NotNull String cleanTripOriginalId(@NotNull String gTripId);
+
 	boolean directionSplitterEnabled(long routeId);
 
 	boolean directionOverrideId(long routeId);

src/main/java/org/mtransit/parser/gtfs/GReader.java

@@ -286,7 +286,7 @@ private static void readCsv(String filename, BufferedReader reader,
 	private static void processStopTime(GAgencyTools agencyTools, GSpec gSpec, HashMap<String, String> line,
 										@Nullable PreparedStatement insertStopTimePrepared) {
 		try {
-			final GStopTime gStopTime = GStopTime.fromLine(line);
+			final GStopTime gStopTime = GStopTime.fromLine(line, agencyTools);
 			if (agencyTools.excludeTripNullable(gSpec.getTrip(gStopTime.getTripIdInt()))) {
 				return;
 			}
@@ -316,7 +316,7 @@ private static void processFrequency(GAgencyTools agencyTools,
 										 GSpec gSpec,
 										 HashMap<String, String> line) {
 		try {
-			final GFrequency gFrequency = GFrequency.fromLine(line);
+			final GFrequency gFrequency = GFrequency.fromLine(line, agencyTools);
 			if (agencyTools.excludeTripNullable(gSpec.getTrip(gFrequency.getTripIdInt()))) {
 				return;
 			}
@@ -338,7 +338,7 @@ private static void processAgency(GAgencyTools agencyTools,
 			agencyTools.addSupportedLanguage(gAgency.getAgencyLang());
 			gSpec.addAgency(gAgency);
 		} catch (Exception e) {
-			throw new MTLog.Fatal(e, "Error while processing: '%s'!", line);
+			throw new MTLog.Fatal(e, "Error while processing agency: '%s'!", line);
 		}
 	}
 
@@ -354,7 +354,7 @@ private static void processCalendarDate(GAgencyTools agencyTools, GSpec gSpec, H
 			}
 			gSpec.addCalendarDate(gCalendarDate);
 		} catch (Exception e) {
-			throw new MTLog.Fatal(e, "Error while processing: '%s'!", line);
+			throw new MTLog.Fatal(e, "Error while processing calendar date: '%s'!", line);
 		}
 	}
 
@@ -366,13 +366,13 @@ private static void processCalendar(GAgencyTools agencyTools, GSpec gSpec, HashM
 			}
 			gSpec.addCalendar(gCalendar);
 		} catch (Exception e) {
-			throw new MTLog.Fatal(e, "Error while processing: %s!", line);
+			throw new MTLog.Fatal(e, "Error while processing calendar: %s!", line);
 		}
 	}
 
 	private static void processDirection(GAgencyTools agencyTools, GSpec gSpec, HashMap<String, String> line) {
 		try {
-			final GDirection gDirection = GDirection.fromLine(line);
+			final GDirection gDirection = GDirection.fromLine(line, agencyTools);
 			final GRoute gRoute = gSpec.getRoute(gDirection.getRouteIdInt());
 			if (agencyTools.excludeRouteNullable(gRoute)) {
 				logExclude("Exclude route: %s.", gRoute == null ? null : gRoute.toStringPlus());
@@ -393,7 +393,7 @@ private static void processDirection(GAgencyTools agencyTools, GSpec gSpec, Hash
 	private static void processTrip(GAgencyTools agencyTools, GSpec gSpec, HashMap<String, String> line,
 									@Nullable PreparedStatement insertStopTimePrepared) {
 		try {
-			final GTrip gTrip = GTrip.fromLine(line);
+			final GTrip gTrip = GTrip.fromLine(line, agencyTools);
 			if (agencyTools.excludeTrip(gTrip)) {
 				logExclude("Exclude trip: %s.", gTrip.toStringPlus());
 				return;
@@ -411,16 +411,22 @@ private static void processTrip(GAgencyTools agencyTools, GSpec gSpec, HashMap<S
 			}
 			gSpec.addTrip(gTrip, insertStopTimePrepared);
 		} catch (Exception e) {
-			throw new MTLog.Fatal(e, "Error while processing: %s", line);
+			throw new MTLog.Fatal(e, "Error while processing trip: %s", line);
 		}
 	}
 
 	private static void processStop(GAgencyTools agencyTools, GSpec gSpec, Map<String, String> line) {
 		try {
-			final GStop gStop = GStop.fromLine(line);
+			final GStop gStop = GStop.fromLine(line, agencyTools);
 			if (agencyTools.excludeStop(gStop)) {
 				return;
 			}
+			if (agencyTools.getStopIdCleanupRegex() != null) { // IF stop ID cleanup regex set DO
+				final GStop previousStop = gSpec.getStop(gStop.getStopIdInt());
+				if (previousStop != null && previousStop.equals(gStop)) {
+					return; // ignore if stop already exists with same values
+				}
+			}
 			gSpec.addStop(gStop);
 		} catch (Exception e) {
 			throw new MTLog.Fatal(e, "Error while parsing stop line %s!", line);
@@ -429,7 +435,7 @@ private static void processStop(GAgencyTools agencyTools, GSpec gSpec, Map<Strin
 
 	private static void processRoute(GAgencyTools agencyTools, GSpec gSpec, HashMap<String, String> line, @Nullable String defaultAgencyId) {
 		try {
-			final GRoute gRoute = GRoute.fromLine(line, defaultAgencyId);
+			final GRoute gRoute = GRoute.fromLine(line, defaultAgencyId, agencyTools);
 			final GAgency routeAgency = gSpec.getAgency(gRoute.getAgencyIdInt());
 			if (agencyTools.excludeRoute(gRoute)) {
 				logExclude("Exclude route: %s.", gRoute.toStringPlus());
@@ -448,6 +454,22 @@ private static void processRoute(GAgencyTools agencyTools, GSpec gSpec, HashMap<
 				}
 				return;
 			}
+			if (agencyTools.getRouteIdCleanupRegex() != null) { // IF route ID cleanup regex set DO
+				final GRoute previousRoute = gSpec.getRoute(gRoute.getRouteIdInt());
+				if (previousRoute != null && previousRoute.equals(gRoute)) {
+					return; // ignore if route already exists with same values
+				}
+				if (previousRoute != null && previousRoute.equalsExceptLongNameAndUrl(gRoute)) {
+					final String mergedRouteLongName = GRoute.mergeRouteLongNames(previousRoute.getRouteLongName(), gRoute.getRouteLongName());
+					if (mergedRouteLongName != null) { // merge successful
+						gSpec.addRoute(previousRoute.clone(mergedRouteLongName), true);
+						return;
+					}
+				}
+				if (previousRoute != null) {
+					MTLog.log("Duplicate route ID!\n-%s\n-%s", gRoute.toStringPlus(), previousRoute.toStringPlus());
+				}
+			}
 			gSpec.addRoute(gRoute);
 		} catch (Exception e) {
 			throw new MTLog.Fatal(e, "Error while parsing route line %s!", line);

src/main/java/org/mtransit/parser/gtfs/data/GDirection.kt

@@ -1,9 +1,11 @@
 package org.mtransit.parser.gtfs.data
 
 import androidx.annotation.Discouraged
+import org.mtransit.commons.GTFSCommons
 import org.mtransit.commons.gtfs.data.Direction
 import org.mtransit.commons.gtfs.data.DirectionType
 import org.mtransit.parser.MTLog
+import org.mtransit.parser.gtfs.GAgencyTools
 
 data class GDirection(
     val routeIdInt: Int,
@@ -48,7 +50,7 @@ data class GDirection(
     companion object {
         const val FILENAME = "directions.txt"
 
-        private const val ROUTE_ID = "route_id"
+        private const val ROUTE_ID = GRoute.ROUTE_ID
         private const val DIRECTION_ID = "direction_id"
         private const val DIRECTION = "direction"
 
@@ -57,8 +59,10 @@ data class GDirection(
         // TODO other alternatives
 
         @JvmStatic
-        fun fromLine(line: Map<String, String>) = GDirection(
-            routeId = line[ROUTE_ID] ?: throw MTLog.Fatal("Invalid GDirection from $line!"),
+        fun fromLine(line: Map<String, String>, agencyTools: GAgencyTools) = GDirection(
+            routeId = line[ROUTE_ID]?.trim()
+                ?.let { agencyTools.cleanRouteOriginalId(it) }
+                ?: throw MTLog.Fatal("Invalid GDirection from $line!"),
             directionIdInt = line[DIRECTION_ID]?.toIntOrNull() ?: throw MTLog.Fatal("Invalid GDirection from $line!"),
             directionTypeValue = line[DIRECTION]?.trim(),
             destination = line[DESTINATION]?.trim() ?: line[DESTINATION_DIRECTION_NAME]?.trim(),

src/main/java/org/mtransit/parser/gtfs/data/GFrequency.kt

@@ -3,6 +3,7 @@ package org.mtransit.parser.gtfs.data
 import androidx.annotation.Discouraged
 import org.mtransit.commons.gtfs.data.Frequency
 import org.mtransit.parser.MTLog
+import org.mtransit.parser.gtfs.GAgencyTools
 import java.util.Date
 import java.util.concurrent.TimeUnit
 
@@ -88,7 +89,7 @@ data class GFrequency(
     companion object {
         const val FILENAME = "frequencies.txt"
 
-        private const val TRIP_ID = "trip_id"
+        private const val TRIP_ID = GTrip.TRIP_ID
         private const val START_TIME = "start_time"
         private const val END_TIME = "end_time"
         private const val HEADWAY_SECS = "headway_secs"
@@ -101,12 +102,14 @@ data class GFrequency(
         val DEFAULT_DROP_OFF_TYPE = GDropOffType.REGULAR // Regularly scheduled drop off
 
         @JvmStatic
-        fun fromLine(line: Map<String, String>) = GFrequency(
-            line[TRIP_ID] ?: throw MTLog.Fatal("Invalid GFrequency from $line!"),
-            line[START_TIME] ?: throw MTLog.Fatal("Invalid GFrequency from $line!"),
-            line[END_TIME] ?: throw MTLog.Fatal("Invalid GFrequency from $line!"),
-            line[HEADWAY_SECS]?.toInt() ?: throw MTLog.Fatal("Invalid GFrequency from $line!"),
-            line[EXACT_TIMES]?.toIntOrNull(),
+        fun fromLine(line: Map<String, String>, agencyTools: GAgencyTools) = GFrequency(
+            tripId = line[TRIP_ID]?.trim()
+                ?.let { agencyTools.cleanTripOriginalId(it) }
+                ?: throw MTLog.Fatal("Invalid GFrequency from $line!"),
+            startTime = line[START_TIME] ?: throw MTLog.Fatal("Invalid GFrequency from $line!"),
+            endTime = line[END_TIME] ?: throw MTLog.Fatal("Invalid GFrequency from $line!"),
+            headwaySecs = line[HEADWAY_SECS]?.toInt() ?: throw MTLog.Fatal("Invalid GFrequency from $line!"),
+            exactTimes = line[EXACT_TIMES]?.toIntOrNull(),
         )
 
         @JvmStatic

src/main/java/org/mtransit/parser/gtfs/data/GRoute.kt

@@ -6,6 +6,8 @@ import org.mtransit.commons.gtfs.data.Route
 import org.mtransit.commons.gtfs.data.RouteId
 import org.mtransit.parser.Constants.EMPTY
 import org.mtransit.parser.MTLog
+import org.mtransit.parser.gtfs.GAgencyTools
+import kotlin.math.max
 
 // https://developers.google.com/transit/gtfs/reference#routestxt
 // https://gtfs.org/reference/static/#routestxt
@@ -82,11 +84,7 @@ data class GRoute(
         }
 
     @Suppress("unused")
-    val shortestRouteName = if (routeShortName.isEmpty()) {
-        routeLongName
-    } else {
-        routeShortName
-    }
+    val shortestRouteName = routeShortName.ifEmpty { routeLongName }
 
     @Suppress("unused")
     val longestRouteName = if (routeLongName.isNullOrEmpty()) {
@@ -127,11 +125,30 @@ data class GRoute(
         routeSortOrder = routeSortOrder,
     )
 
+    fun equalsExceptLongNameAndUrl(obj: Any?): Boolean {
+        val o = obj as GRoute
+        return when {
+            agencyIdInt != o.agencyIdInt -> false // not equal
+            routeIdInt != o.routeIdInt -> false // not equal
+            routeShortName != o.routeShortName -> false // not equal
+            routeDesc != o.routeDesc -> false // not equal
+            routeType != o.routeType -> false // not equal
+            routeColor != o.routeColor -> false // not equal
+            routeTextColor != o.routeTextColor -> false // not equal
+            routeSortOrder != o.routeSortOrder -> false // not equal
+            else -> true // mostly equal
+        }
+    }
+
+    fun clone(routeLongName: String?) = this.copy(
+        routeLongName = routeLongName,
+    )
+
     companion object {
         const val FILENAME = "routes.txt"
 
         private const val AGENCY_ID = "agency_id"
-        private const val ROUTE_ID = "route_id"
+        internal const val ROUTE_ID = "route_id"
         private const val ROUTE_SHORT_NAME = "route_short_name"
         private const val ROUTE_LONG_NAME = "route_long_name"
         private const val ROUTE_DESC = "route_desc"
@@ -142,11 +159,13 @@ data class GRoute(
         private const val ROUTE_SORT_ORDER = "route_sort_order"
 
         @JvmStatic
-        fun fromLine(line: Map<String, String>, defaultAgencyId: String?) = GRoute(
+        fun fromLine(line: Map<String, String>, defaultAgencyId: String?, agencyTools: GAgencyTools) = GRoute(
             agencyId = line[AGENCY_ID]?.takeIf { it.isNotBlank() }
                 ?: defaultAgencyId
                 ?: throw MTLog.Fatal("Invalid GRoute.$AGENCY_ID from $line!"),
-            routeId = line[ROUTE_ID] ?: throw MTLog.Fatal("Invalid GRoute.$ROUTE_ID from $line!"),
+            routeId = line[ROUTE_ID]?.trim()
+                ?.let { agencyTools.cleanRouteOriginalId(it) }
+                ?: throw MTLog.Fatal("Invalid GRoute.$ROUTE_ID from $line!"),
             routeShortName = line[ROUTE_SHORT_NAME]?.trim() ?: EMPTY,
             routeLongName = line[ROUTE_LONG_NAME],
             routeDesc = line[ROUTE_DESC],
@@ -175,5 +194,40 @@ data class GRoute(
                 routeSortOrder = it.routeSortOrder,
             )
         }
+
+        private const val SLASH_: String = " / "
+
+        @JvmStatic
+        fun mergeRouteLongNames(routeLongName1: String?, routeLongName2: String?): String? {
+            if (routeLongName2.isNullOrEmpty()) {
+                return routeLongName1
+            } else if (routeLongName1.isNullOrEmpty()) {
+                return routeLongName2
+            } else if (routeLongName2.contains(routeLongName1)) {
+                return routeLongName2
+            } else if (routeLongName1.contains(routeLongName2)) {
+                return routeLongName1
+            }
+            val prefix = routeLongName1.commonPrefixWith(routeLongName2)
+            val maxLength = max(routeLongName1.length, routeLongName2.length)
+            if (prefix.length > maxLength / 2) {
+                return prefix +
+                        routeLongName1.substring(prefix.length) +
+                        SLASH_ +
+                        routeLongName2.substring(prefix.length)
+            }
+            val suffix = routeLongName1.commonSuffixWith(routeLongName2)
+            if (suffix.length > maxLength / 2) {
+                return routeLongName1.substring(0, routeLongName1.length - suffix.length) +
+                        SLASH_ +
+                        routeLongName2.substring(0, routeLongName2.length - suffix.length) +
+                        suffix
+            }
+            return if (routeLongName1 > routeLongName2) {
+                routeLongName2 + SLASH_ + routeLongName1
+            } else {
+                routeLongName1 + SLASH_ + routeLongName2
+            }
+        }
     }
 }