From 23ab23e78432e5cefe87d4960c78a17b12d910ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20M=C3=A9a?= Date: Tue, 3 Feb 2026 09:36:17 -0500 Subject: [PATCH 1/8] Remove unused trip IDs ints. --- .../mtransit/parser/DefaultAgencyTools.java | 1 + .../org/mtransit/parser/gtfs/data/GSpec.java | 19 +++++++++++++++++++ .../org/mtransit/parser/mt/data/MTripIds.kt | 10 +++++++++- 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/mtransit/parser/DefaultAgencyTools.java b/src/main/java/org/mtransit/parser/DefaultAgencyTools.java index d869044..1f3ffc1 100644 --- a/src/main/java/org/mtransit/parser/DefaultAgencyTools.java +++ b/src/main/java/org/mtransit/parser/DefaultAgencyTools.java @@ -169,6 +169,7 @@ public void start(@NotNull String[] args) { gtfs.cleanupStops(); gtfs.cleanupExcludedData(); gtfs.cleanupStopTimesPickupDropOffTypes(this); + gtfs.cleanupIds(); gtfs.generateTripStops(); if (args.length >= 4 && Boolean.parseBoolean(args[3])) { gtfs.generateStopTimesFromFrequencies(this); diff --git a/src/main/java/org/mtransit/parser/gtfs/data/GSpec.java b/src/main/java/org/mtransit/parser/gtfs/data/GSpec.java index 33d974f..14df20a 100644 --- a/src/main/java/org/mtransit/parser/gtfs/data/GSpec.java +++ b/src/main/java/org/mtransit/parser/gtfs/data/GSpec.java @@ -12,6 +12,8 @@ import org.mtransit.parser.db.GTFSDataBase; import org.mtransit.parser.gtfs.GAgencyTools; import org.mtransit.parser.mt.GenerateMObjectsTask; +import org.mtransit.parser.mt.data.MTripId; +import org.mtransit.parser.mt.data.MTripIds; import java.sql.PreparedStatement; import java.util.ArrayList; @@ -326,6 +328,7 @@ public Collection getAllStops() { return this.stopsCache.values(); } + @SuppressWarnings("unused") @NotNull private Collection getAllStopIds() { if (USE_DB_ONLY) { @@ -823,6 +826,22 @@ public GSpec getRouteGTFS(@SuppressWarnings("unused") @NotNull Long mRouteId) { return this; } + public void cleanupIds() { + MTLog.log("Removing unused IDs..."); + int r = 0; + try { + for (MTripId mTripId : MTripIds.getAll()) { + if (getTripRouteId(GIDs.getInt(mTripId.getTripId())) == null) { + MTripIds.remove(mTripId); + r++; + } + } + } catch (Exception e) { + throw new MTLog.Fatal(e, "Error while removing unused IDs!"); + } + MTLog.log("Removing unused IDs... DONE (%d removed objects)", r); + } + public void cleanupExcludedData() { MTLog.log("Removing more excluded data..."); int r = 0; diff --git a/src/main/java/org/mtransit/parser/mt/data/MTripIds.kt b/src/main/java/org/mtransit/parser/mt/data/MTripIds.kt index a8116f9..6de5ce0 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MTripIds.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MTripIds.kt @@ -36,6 +36,14 @@ object MTripIds { } } + @JvmStatic + fun remove(tripId: MTripId) { + synchronized(incrementLock) { + idIntToId.remove(tripId.tripIdInt) + idToIdInt.remove(tripId.tripId) + } + } + @Suppress("unused") @JvmStatic fun getId(tripIdInt: Int) = @@ -58,7 +66,7 @@ object MTripIds { } }.sorted() - @JvmStatic + @JvmStatic fun convert(tripId: String) = if (FeatureFlags.F_EXPORT_TRIP_ID_INTS) { getInt(tripId).toString() From 7057c06681fc764aca5e876b06e30dee803719f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20M=C3=A9a?= Date: Tue, 3 Feb 2026 10:04:44 -0500 Subject: [PATCH 2/8] fix --- .../mtransit/parser/DefaultAgencyTools.java | 1 - .../org/mtransit/parser/gtfs/data/GSpec.java | 18 ----------- .../org/mtransit/parser/mt/MGenerator.java | 32 ++++++++++++++++--- .../org/mtransit/parser/mt/data/MTripIds.kt | 5 ++- 4 files changed, 31 insertions(+), 25 deletions(-) diff --git a/src/main/java/org/mtransit/parser/DefaultAgencyTools.java b/src/main/java/org/mtransit/parser/DefaultAgencyTools.java index 1f3ffc1..d869044 100644 --- a/src/main/java/org/mtransit/parser/DefaultAgencyTools.java +++ b/src/main/java/org/mtransit/parser/DefaultAgencyTools.java @@ -169,7 +169,6 @@ public void start(@NotNull String[] args) { gtfs.cleanupStops(); gtfs.cleanupExcludedData(); gtfs.cleanupStopTimesPickupDropOffTypes(this); - gtfs.cleanupIds(); gtfs.generateTripStops(); if (args.length >= 4 && Boolean.parseBoolean(args[3])) { gtfs.generateStopTimesFromFrequencies(this); diff --git a/src/main/java/org/mtransit/parser/gtfs/data/GSpec.java b/src/main/java/org/mtransit/parser/gtfs/data/GSpec.java index 14df20a..7ef497c 100644 --- a/src/main/java/org/mtransit/parser/gtfs/data/GSpec.java +++ b/src/main/java/org/mtransit/parser/gtfs/data/GSpec.java @@ -12,8 +12,6 @@ import org.mtransit.parser.db.GTFSDataBase; import org.mtransit.parser.gtfs.GAgencyTools; import org.mtransit.parser.mt.GenerateMObjectsTask; -import org.mtransit.parser.mt.data.MTripId; -import org.mtransit.parser.mt.data.MTripIds; import java.sql.PreparedStatement; import java.util.ArrayList; @@ -826,22 +824,6 @@ public GSpec getRouteGTFS(@SuppressWarnings("unused") @NotNull Long mRouteId) { return this; } - public void cleanupIds() { - MTLog.log("Removing unused IDs..."); - int r = 0; - try { - for (MTripId mTripId : MTripIds.getAll()) { - if (getTripRouteId(GIDs.getInt(mTripId.getTripId())) == null) { - MTripIds.remove(mTripId); - r++; - } - } - } catch (Exception e) { - throw new MTLog.Fatal(e, "Error while removing unused IDs!"); - } - MTLog.log("Removing unused IDs... DONE (%d removed objects)", r); - } - public void cleanupExcludedData() { MTLog.log("Removing more excluded data..."); int r = 0; diff --git a/src/main/java/org/mtransit/parser/mt/MGenerator.java b/src/main/java/org/mtransit/parser/mt/MGenerator.java index 43c3ebc..e9270b6 100644 --- a/src/main/java/org/mtransit/parser/mt/MGenerator.java +++ b/src/main/java/org/mtransit/parser/mt/MGenerator.java @@ -23,6 +23,7 @@ import org.mtransit.parser.gtfs.GAgencyTools; import org.mtransit.parser.gtfs.GReader; import org.mtransit.parser.gtfs.data.GFieldTypes; +import org.mtransit.parser.gtfs.data.GIDs; import org.mtransit.parser.gtfs.data.GSpec; import org.mtransit.parser.mt.data.MAgency; import org.mtransit.parser.mt.data.MFrequency; @@ -80,8 +81,8 @@ public static MSpec generateMSpec(@NotNull GSpec gtfs, @NotNull GAgencyTools age HashSet mRoutes = new HashSet<>(); // use set to avoid duplicates HashSet mDirections = new HashSet<>(); // use set to avoid duplicates HashSet mDirectionStops = new HashSet<>(); // use set to avoid duplicates - HashSet mTrips = new HashSet<>(); // use set to avoid duplicates - HashMap mStops = new HashMap<>(); + HashMap mTrips = new HashMap<>(); // map key unique -> avoid duplicates + HashMap mStops = new HashMap<>(); // map key unique -> avoid duplicates TreeMap> mRouteFrequencies = new TreeMap<>(); HashSet mServiceDates = new HashSet<>(); // use set to avoid duplicates long firstTimestamp = -1L; @@ -107,7 +108,15 @@ public static MSpec generateMSpec(@NotNull GSpec gtfs, @NotNull GAgencyTools age mRoutes.addAll(mRouteSpec.getRoutes()); mDirections.addAll(mRouteSpec.getDirections()); mDirectionStops.addAll(mRouteSpec.getDirectionStops()); - mTrips.addAll(mRouteSpec.getTrips()); + for (MTrip mTrip : mRouteSpec.getTrips()) { + if (mTrips.containsKey(mTrip.getTripIdInt())) { + if (!mTrips.get(mTrip.getTripIdInt()).equals(mTrip)) { + MTLog.log("%s: Trip ID '%s' already in list! (%s instead of %s)", mRouteId, mTrip.getTripId(), mTrips.get(mTrip.getTripId()), mTrip); + } + continue; + } + mTrips.put(mTrip.getTripIdInt(), mTrip); + } logMerging("stops...", mRouteId); for (MStop mStop : mRouteSpec.getStops()) { if (mStops.containsKey(mStop.getId())) { @@ -164,13 +173,26 @@ public static MSpec generateMSpec(@NotNull GSpec gtfs, @NotNull GAgencyTools age } MTLog.log("Generating routes, trips, trip stops & stops objects... (all routes completed)"); threadPoolExecutor.shutdown(); + MTLog.log("Removing unsued IDs..."); + int r = 0; + try { + for (MTripId mTripId : MTripIds.getAll()) { + if (!mTrips.containsKey(GIDs.getInt(mTripId.getTripId()))) { + MTripIds.remove(mTripId); + r++; + } + } + } catch (Exception e) { + throw new MTLog.Fatal(e, "Error while removing unused IDs!"); + } + MTLog.log("Removing unused IDs... DONE (%d removed objects)", r); final ArrayList mAgenciesList = new ArrayList<>(mAgencies); Collections.sort(mAgenciesList); final ArrayList mStopsList = new ArrayList<>(mStops.values()); Collections.sort(mStopsList); final ArrayList mRoutesList = new ArrayList<>(mRoutes); Collections.sort(mRoutesList); - final ArrayList mTripsList = new ArrayList<>(mTrips); + final ArrayList mTripsList = new ArrayList<>(mTrips.values()); Collections.sort(mTripsList); final ArrayList mDirectionsList = new ArrayList<>(mDirections); Collections.sort(mDirectionsList); @@ -629,7 +651,7 @@ private static void dumpTripIds( dbStatement = dbConnection.createStatement(); sqlInsert = GTFSCommons.getT_TRIP_IDS_SQL_INSERT(); } - for (MTripId mTripId : MTripIds.getAll()) { + for (MTripId mTripId : MTripIds.getAllSorted()) { final String tripIdsInsert = mTripId.toFile(); if (FeatureFlags.F_PRE_FILLED_DB) { SQLUtils.executeUpdate( diff --git a/src/main/java/org/mtransit/parser/mt/data/MTripIds.kt b/src/main/java/org/mtransit/parser/mt/data/MTripIds.kt index 6de5ce0..ac213f2 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MTripIds.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MTripIds.kt @@ -64,7 +64,10 @@ object MTripIds { idToIdInt.forEach { id, idInt -> add(MTripId(idInt, id)) } - }.sorted() + } + + @JvmStatic + fun getAllSorted() = getAll().sorted() @JvmStatic fun convert(tripId: String) = From b7e77aa6248c134e8ed50d6ed334ebef8e0fa181 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20M=C3=A9a?= Date: Tue, 3 Feb 2026 10:05:34 -0500 Subject: [PATCH 3/8] fix --- src/main/java/org/mtransit/parser/mt/MGenerator.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/mtransit/parser/mt/MGenerator.java b/src/main/java/org/mtransit/parser/mt/MGenerator.java index e9270b6..d6cb4dd 100644 --- a/src/main/java/org/mtransit/parser/mt/MGenerator.java +++ b/src/main/java/org/mtransit/parser/mt/MGenerator.java @@ -111,7 +111,8 @@ public static MSpec generateMSpec(@NotNull GSpec gtfs, @NotNull GAgencyTools age for (MTrip mTrip : mRouteSpec.getTrips()) { if (mTrips.containsKey(mTrip.getTripIdInt())) { if (!mTrips.get(mTrip.getTripIdInt()).equals(mTrip)) { - MTLog.log("%s: Trip ID '%s' already in list! (%s instead of %s)", mRouteId, mTrip.getTripId(), mTrips.get(mTrip.getTripId()), mTrip); + //noinspection DiscouragedApi + MTLog.log("%s: Trip ID '%s' already in list! (%s instead of %s)", mRouteId, mTrip.getTripId(), mTrips.get(mTrip.getTripIdInt()), mTrip); } continue; } From 8ed1815457328a2c44e9030f33e7489c5562fe30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20M=C3=A9a?= Date: Tue, 3 Feb 2026 10:31:11 -0500 Subject: [PATCH 4/8] PR comments --- .../java/org/mtransit/parser/mt/MGenerator.java | 11 +++-------- .../java/org/mtransit/parser/mt/data/MTripIds.kt | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/mtransit/parser/mt/MGenerator.java b/src/main/java/org/mtransit/parser/mt/MGenerator.java index d6cb4dd..538e0e1 100644 --- a/src/main/java/org/mtransit/parser/mt/MGenerator.java +++ b/src/main/java/org/mtransit/parser/mt/MGenerator.java @@ -174,15 +174,10 @@ public static MSpec generateMSpec(@NotNull GSpec gtfs, @NotNull GAgencyTools age } MTLog.log("Generating routes, trips, trip stops & stops objects... (all routes completed)"); threadPoolExecutor.shutdown(); - MTLog.log("Removing unsued IDs..."); - int r = 0; + MTLog.log("Removing unused IDs..."); + int r; try { - for (MTripId mTripId : MTripIds.getAll()) { - if (!mTrips.containsKey(GIDs.getInt(mTripId.getTripId()))) { - MTripIds.remove(mTripId); - r++; - } - } + r = MTripIds.prune(GIDs.getStrings(mTrips.keySet())); } catch (Exception e) { throw new MTLog.Fatal(e, "Error while removing unused IDs!"); } diff --git a/src/main/java/org/mtransit/parser/mt/data/MTripIds.kt b/src/main/java/org/mtransit/parser/mt/data/MTripIds.kt index ac213f2..01e4630 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MTripIds.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MTripIds.kt @@ -36,6 +36,21 @@ object MTripIds { } } + @JvmStatic + fun prune(usedTripIds: Iterable): Int { + var removedCount = 0 + synchronized(incrementLock) { + idToIdInt.asMap() + .filter { it.key !in usedTripIds } + .forEach { (tripId, tripIdInt) -> + idIntToId.remove(tripIdInt) + idToIdInt.remove(tripId) + removedCount++ + } + } + return removedCount + } + @JvmStatic fun remove(tripId: MTripId) { synchronized(incrementLock) { From 2e30cc1a10af81a378f781bde99ca9fb20279999 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20M=C3=A9a?= Date: Tue, 3 Feb 2026 10:39:30 -0500 Subject: [PATCH 5/8] PR comments --- .../org/mtransit/parser/mt/MGenerator.java | 22 ++++++++----------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/mtransit/parser/mt/MGenerator.java b/src/main/java/org/mtransit/parser/mt/MGenerator.java index 538e0e1..de59df4 100644 --- a/src/main/java/org/mtransit/parser/mt/MGenerator.java +++ b/src/main/java/org/mtransit/parser/mt/MGenerator.java @@ -108,25 +108,21 @@ public static MSpec generateMSpec(@NotNull GSpec gtfs, @NotNull GAgencyTools age mRoutes.addAll(mRouteSpec.getRoutes()); mDirections.addAll(mRouteSpec.getDirections()); mDirectionStops.addAll(mRouteSpec.getDirectionStops()); + logMerging("trips...", mRouteId); for (MTrip mTrip : mRouteSpec.getTrips()) { - if (mTrips.containsKey(mTrip.getTripIdInt())) { - if (!mTrips.get(mTrip.getTripIdInt()).equals(mTrip)) { - //noinspection DiscouragedApi - MTLog.log("%s: Trip ID '%s' already in list! (%s instead of %s)", mRouteId, mTrip.getTripId(), mTrips.get(mTrip.getTripIdInt()), mTrip); - } - continue; + final MTrip existing = mTrips.putIfAbsent(mTrip.getTripIdInt(), mTrip); + if (existing != null && !existing.equals(mTrip)) { + //noinspection DiscouragedApi + MTLog.log("%s: Trip ID '%s' already in list! (%s instead of %s)", mRouteId, mTrip.getTripId(), existing.toStringPlus(), mTrip.toStringPlus()); } - mTrips.put(mTrip.getTripIdInt(), mTrip); } + logMerging("trips... DONE", mRouteId); logMerging("stops...", mRouteId); for (MStop mStop : mRouteSpec.getStops()) { - if (mStops.containsKey(mStop.getId())) { - if (!mStops.get(mStop.getId()).equals(mStop)) { - MTLog.log("%s: Stop ID '%s' already in list! (%s instead of %s)", mRouteId, mStop.getId(), mStops.get(mStop.getId()), mStop); - } - continue; + final MStop existing = mStops.putIfAbsent(mStop.getId(), mStop); + if (existing != null && !existing.equals(mStop)) { + MTLog.log("%s: Stop ID '%s' already in list! (%s instead of %s)", mRouteId, mStop.getId(), existing, mStop); } - mStops.put(mStop.getId(), mStop); } logMerging("stops... DONE", mRouteId); logMerging("service dates...", mRouteId); From 8b911e5169c01b7d69a28ff75b25218199d2d07c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20M=C3=A9a?= Date: Tue, 3 Feb 2026 10:40:03 -0500 Subject: [PATCH 6/8] cleanup --- src/main/java/org/mtransit/parser/mt/data/MTripIds.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/mtransit/parser/mt/data/MTripIds.kt b/src/main/java/org/mtransit/parser/mt/data/MTripIds.kt index 01e4630..cc65a77 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MTripIds.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MTripIds.kt @@ -51,6 +51,7 @@ object MTripIds { return removedCount } + @Suppress("unused") @JvmStatic fun remove(tripId: MTripId) { synchronized(incrementLock) { From 73616a14e66119e8df2b2eddd5480e0e008bcd2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20M=C3=A9a?= Date: Tue, 3 Feb 2026 10:41:12 -0500 Subject: [PATCH 7/8] cleanup --- src/main/java/org/mtransit/parser/mt/data/MTripIds.kt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/mtransit/parser/mt/data/MTripIds.kt b/src/main/java/org/mtransit/parser/mt/data/MTripIds.kt index cc65a77..931069e 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MTripIds.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MTripIds.kt @@ -41,10 +41,10 @@ object MTripIds { var removedCount = 0 synchronized(incrementLock) { idToIdInt.asMap() - .filter { it.key !in usedTripIds } - .forEach { (tripId, tripIdInt) -> - idIntToId.remove(tripIdInt) - idToIdInt.remove(tripId) + .filter { (tripId, _) -> tripId !in usedTripIds } + .forEach { (unusedTripId, unusedTripIdInt) -> + idIntToId.remove(unusedTripIdInt) + idToIdInt.remove(unusedTripId) removedCount++ } } From 52c821ffe5b44d787b93183c0be49a275c900180 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20M=C3=A9a?= Date: Tue, 3 Feb 2026 10:59:18 -0500 Subject: [PATCH 8/8] PR comment --- .../java/org/mtransit/parser/mt/MGenerator.java | 2 +- .../java/org/mtransit/parser/mt/data/MTripIds.kt | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/mtransit/parser/mt/MGenerator.java b/src/main/java/org/mtransit/parser/mt/MGenerator.java index de59df4..8d0dfe3 100644 --- a/src/main/java/org/mtransit/parser/mt/MGenerator.java +++ b/src/main/java/org/mtransit/parser/mt/MGenerator.java @@ -173,7 +173,7 @@ public static MSpec generateMSpec(@NotNull GSpec gtfs, @NotNull GAgencyTools age MTLog.log("Removing unused IDs..."); int r; try { - r = MTripIds.prune(GIDs.getStrings(mTrips.keySet())); + r = MTripIds.prune(new HashSet<>(GIDs.getStrings(mTrips.keySet()))); } catch (Exception e) { throw new MTLog.Fatal(e, "Error while removing unused IDs!"); } diff --git a/src/main/java/org/mtransit/parser/mt/data/MTripIds.kt b/src/main/java/org/mtransit/parser/mt/data/MTripIds.kt index 931069e..426c818 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MTripIds.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MTripIds.kt @@ -37,16 +37,18 @@ object MTripIds { } @JvmStatic - fun prune(usedTripIds: Iterable): Int { + fun prune(usedTripIds: Set): Int { var removedCount = 0 synchronized(incrementLock) { - idToIdInt.asMap() - .filter { (tripId, _) -> tripId !in usedTripIds } - .forEach { (unusedTripId, unusedTripIdInt) -> - idIntToId.remove(unusedTripIdInt) - idToIdInt.remove(unusedTripId) + val iterator = idToIdInt.asMutableMap().entries.iterator() + while (iterator.hasNext()) { + val (tripId,tripIdInt) = iterator.next() + if (tripId !in usedTripIds) { + idIntToId.remove(tripIdInt) + iterator.remove() removedCount++ } + } } return removedCount }