Allow ignore invalid times

src/main/java/org/mtransit/parser/DefaultAgencyTools.java

@@ -591,7 +591,7 @@ public String cleanRouteLongName(@NotNull String routeLongName) {
 		}
 		routeLongName = Configs.getRouteConfig().cleanRouteLongName(routeLongName);
 		if (defaultStringsCleanerEnabled()) {
-			return StringsCleaner.cleanRouteLongName(routeLongName, getSupportedLanguages(), lowerUCStrings(), lowerUCWords(),  getIgnoreUCWords());
+			return StringsCleaner.cleanRouteLongName(routeLongName, getSupportedLanguages(), lowerUCStrings(), lowerUCWords(), getIgnoreUCWords());
 		}
 		return org.mtransit.commons.CleanUtils.cleanLabel(getFirstLanguageNN(), routeLongName);
 	}
@@ -612,7 +612,6 @@ public boolean lowerUCStrings() {
 		return false; // OPT-IN feature
 	}
 
-
 	@Override
 	public @NotNull String[] getIgnoreUCWords() {
 		if (Configs.getAgencyConfig() != null) {
@@ -1284,26 +1283,31 @@ public int getThreadPoolSize() {
 		return THREAD_POOL_SIZE;
 	}
 
-	@NotNull
+	@Nullable
 	@Override
 	public Pair<Integer, Integer> getTimes(@NotNull GStopTime gStopTime,
 										   @NotNull List<GStopTime> tripStopTimes,
 										   @NotNull DateFormat timeFormat) {
-		if (!gStopTime.hasArrivalTime() || !gStopTime.hasDepartureTime()) {
-			return extractTimes(gStopTime, tripStopTimes, timeFormat);
-		} else {
+		if (gStopTime.hasArrivalTime() && gStopTime.hasDepartureTime()) {
 			return new Pair<>(
 					TimeUtils.cleanExtraSeconds(gStopTime.getArrivalTime()),
 					TimeUtils.cleanExtraSeconds(gStopTime.getDepartureTime()));
 		}
+		return extractTimes(gStopTime, tripStopTimes, timeFormat);
 	}
 
-	@NotNull
+	@Override
+	public boolean allowIgnoreInvalidStopTimes() {
+		return Configs.getRouteConfig().allowIgnoreInvalidStopTimes(getTodayDateInt()); // this is bad, some transit agency data can NOT be fixed :(
+	}
+
+	@Nullable
 	private static Pair<Integer, Integer> extractTimes(GStopTime gStopTime,
 													   @NotNull List<GStopTime> tripStopTimes,
 													   DateFormat timeFormat) {
 		try {
 			Pair<Long, Long> timesInMs = extractTimeInMs(gStopTime, tripStopTimes);
+			if (timesInMs == null) return null;
 			long arrivalTimeInMs = timesInMs.first;
 			Calendar calendar = Calendar.getInstance();
 			calendar.setTimeInMillis(arrivalTimeInMs);
@@ -1326,7 +1330,7 @@ private static Pair<Integer, Integer> extractTimes(GStopTime gStopTime,
 		}
 	}
 
-	@NotNull
+	@Nullable
 	public static Pair<Long, Long> extractTimeInMs(@NotNull GStopTime gStopTime,
 												   @NotNull List<GStopTime> tripGStopTimes) {
 		int previousArrivalTime = -1;
@@ -1393,7 +1397,8 @@ public static Pair<Long, Long> extractTimeInMs(@NotNull GStopTime gStopTime,
 				MTLog.log("- %s", aStopTime.toStringPlus(true));
 			}
 			//noinspection DiscouragedApi
-			throw new MTLog.Fatal("Invalid stop time trip ID '%s' > no previous stop for %s!", gStopTime.getTripId(), gStopTime.toStringPlus(true));
+			MTLog.log("Invalid stop time trip ID '%s' > no previous stop for %s!", gStopTime.getTripId(), gStopTime.toStringPlus(true));
+			return null;
 		}
 		long previousArrivalTimeInMs = GTime.toMs(previousArrivalTime);
 		long previousDepartureTimeInMs = GTime.toMs(previousDepartureTime);
@@ -1404,7 +1409,8 @@ public static Pair<Long, Long> extractTimeInMs(@NotNull GStopTime gStopTime,
 				MTLog.log("- %s", aStopTime.toStringPlus(true));
 			}
 			//noinspection DiscouragedApi
-			throw new MTLog.Fatal("Invalid stop time trip ID '%s' > no next stop for %s!", gStopTime.getTripId(), gStopTime.toStringPlus(true));
+			MTLog.log("Invalid stop time trip ID '%s' > no next stop for %s!", gStopTime.getTripId(), gStopTime.toStringPlus(true));
+			return null;
 		}
 		long nextArrivalTimeInMs = GTime.toMs(nextArrivalTime);
 		long nextDepartureTimeInMs = GTime.toMs(nextDepartureTime);

src/main/java/org/mtransit/parser/config/gtfs/data/RouteConfig.kt

@@ -79,7 +79,11 @@ data class RouteConfig(
     val stopHeadsignRemoveRouteLongName: Boolean = false, // OPT-IN feature
     @SerialName("stop_headsign_cleanup_regex")
     val stopHeadsignCleanupRegex: String? = null, // optional
-
+    // STOP TIMES
+    @SerialName("allow_invalid_stop_times")
+    val allowInvalidStopTimes: Boolean = false, // OPT-IN feature
+    @SerialName("allow_invalid_stop_times_until")
+    val allowInvalidStopTimesUntil: String? = null, // OPT-IN feature
 ) {
 
     @Serializable
@@ -215,4 +219,7 @@ data class RouteConfig(
         }
         return string
     }
+
+    fun allowIgnoreInvalidStopTimes(todayDate: Int) =
+        allowInvalidStopTimes || (allowInvalidStopTimesUntil?.toIntOrNull()?.let { todayDate <= it } ?: false)
 }

src/main/java/org/mtransit/parser/gtfs/GAgencyTools.java

@@ -326,11 +326,13 @@ public interface GAgencyTools {
 	boolean excludeCalendarDate(@NotNull GCalendarDate gCalendarDate);
 
 	// SCHEDULE
-	@NotNull
+	@Nullable
 	Pair<Integer, Integer> getTimes(@NotNull GStopTime gStopTime,
 									@NotNull List<GStopTime> tripStopTimes,
 									@NotNull DateFormat timeFormat);
 
+	boolean allowIgnoreInvalidStopTimes();
+
 	// FREQUENCY
 	int getStartTime(@NotNull GFrequency gFrequency);
 

src/main/java/org/mtransit/parser/gtfs/data/GSpec.java

@@ -798,6 +798,9 @@ private void setDepartureTimeCal(@NotNull Calendar calendar,
 			calendar.setTimeInMillis(gStopTime.getDepartureTimeMs());
 		} else {
 			final Pair<Long, Long> arrivalAndDeparture = DefaultAgencyTools.extractTimeInMs(gStopTime, tripStopTimes);
+			if (arrivalAndDeparture == null) {
+				throw new MTLog.Fatal("setDepartureTimeCal() > cannot extract arrival & departure time from '%s'!", gStopTime.toStringPlus(true));
+			}
 			final long departureTimeInMs = arrivalAndDeparture.second;
 			calendar.setTimeInMillis(departureTimeInMs);
 		}

src/main/java/org/mtransit/parser/gtfs/data/GStopTime.kt

@@ -157,14 +157,27 @@ data class GStopTime(
 
     @JvmOverloads
     @Suppress("unused")
-    fun toStringPlus(debug: Boolean = Constants.DEBUG): String {
-        return if (debug) { // longer
-            return toString() +
-                    "+(tripId:$_tripId)" +
-                    "+(stopId:$_stopId)"
-        } else { // shorter #CI
-            "{t:$_tripId,s:$_stopId,#:$stopSequence}"
-        }
+    fun toStringPlus(debug: Boolean = Constants.DEBUG) = if (debug) { // longer
+        toString() +
+                "+(tripId:$_tripId)" +
+                "+(stopId:$_stopId)"
+    } else { // shorter #CI
+        buildList {
+            add("t:$_tripId")
+            add("s:$_stopId")
+            add("#:$stopSequence")
+            if (hasDepartureTime()) {
+                add("d:${GTime.toString(_departureTime)}")
+            } else if (hasArrivalTime()) {
+                add("a:${GTime.toString(_arrivalTime)}")
+            }
+            if (pickupType != GPickupType.REGULAR) {
+                add("$pickupType")
+            }
+            if (dropOffType != GDropOffType.REGULAR) {
+                add("$dropOffType")
+            }
+        }.joinToString(separator = ",", prefix = "{", postfix = "}")
     }
 
     fun to() = StopTime(

src/main/java/org/mtransit/parser/mt/GenerateMObjectsTask.java

@@ -772,16 +772,20 @@ private String parseGStopTimes(HashMap<String, MSchedule> mSchedules,
 					// TODO later, when UI can display multiple times same stop/POI & schedules are affected to a specific sequence, keep both
 				}
 			}
+			final Pair<Integer, Integer> times = this.agencyTools.getTimes(gStopTime, gTripStopTimes, TIME_FORMAT);
+			if (times == null)  {
+				if (this.agencyTools.allowIgnoreInvalidStopTimes()) {
+					continue; // this is bad, some transit agency data can NOT be fixed :(
+				} else {
+					throw new MTLog.Fatal("Invalid stop time '%s'!", gStopTime.toStringPlus());
+				}
+			}
 			mSchedule = new MSchedule(
 					this.routeId,
 					serviceIdInt,
 					mDirectionId,
 					mStopId,
-					this.agencyTools.getTimes(
-							gStopTime,
-							gTripStopTimes,
-							TIME_FORMAT
-					),
+					times,
 					gStopTime.getTripIdInt(),
 					gTrip.getWheelchairAccessible().getId()
 			);