Index strings

src/main/java/org/mtransit/parser/Constants.java

@@ -22,12 +22,6 @@ public final class Constants {
 
 	public static final char SPACE = ' ';
 
-	public static final char COLUMN_SEPARATOR = ',';
-
-	public static final String COLUMN_SEPARATOR_ = ",";
-
-	public static final char STRING_DELIMITER = '\'';
-
 	public static final String EMPTY = "";
 
 	public static final String SPACE_ = " ";

src/main/java/org/mtransit/parser/DefaultAgencyTools.java

@@ -39,6 +39,8 @@
 import org.mtransit.parser.mt.data.MServiceIds;
 import org.mtransit.parser.mt.data.MSpec;
 import org.mtransit.parser.mt.data.MDirection;
+import org.mtransit.parser.mt.data.MString;
+import org.mtransit.parser.mt.data.MStrings;
 
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
@@ -154,7 +156,9 @@ public void start(@NotNull String[] args) {
 		MTLog.logDebug("Args [%d]: %s.", args.length, Arrays.asList(args));
 		final List<MServiceDate> lastServiceDates = MReader.loadServiceDates(args[2]);
 		final List<MServiceId> lastServiceIds = MReader.loadServiceIds(args[2]);
+		final List<MString> lastStrings = MReader.loadStrings(args[2]);
 		MServiceIds.addAll(lastServiceIds);
+		MStrings.addAll(lastStrings);
 		this.serviceIdInts = extractUsefulServiceIdInts(args, this, true, lastServiceDates);
 		final String inputUrl = args.length >= 5 ? args[4] : null;
 		if (excludingAll()) {

src/main/java/org/mtransit/parser/db/DumpDbUtils.kt

@@ -27,6 +27,7 @@ object DumpDbUtils {
         connection.createStatement().use { statement ->
             SQLUtils.execute(statement, "PRAGMA auto_vacuum = NONE")
             // DROP IF EXIST
+            SQLUtils.executeUpdate(statement, GTFSCommons.T_STRINGS_SQL_DROP)
             SQLUtils.executeUpdate(statement, GTFSCommons.T_DIRECTION_STOPS_SQL_DROP)
             SQLUtils.executeUpdate(statement, GTFSCommons.T_STOP_SQL_DROP)
             SQLUtils.executeUpdate(statement, GTFSCommons.T_DIRECTION_SQL_DROP)
@@ -44,6 +45,7 @@ object DumpDbUtils {
                 SQLUtils.executeUpdate(statement, GTFSCommons.T_SERVICE_IDS_SQL_CREATE)
             }
             SQLUtils.executeUpdate(statement, GTFSCommons.T_SERVICE_DATES_SQL_CREATE)
+            SQLUtils.executeUpdate(statement, GTFSCommons.T_STRINGS_SQL_CREATE)
         }
     }
 }

src/main/java/org/mtransit/parser/db/SQLUtils.kt

@@ -21,6 +21,8 @@ object SQLUtils {
     private const val QUOTE_ = '\''
     private const val QUOTE = SQLUtilsCommons.STRING_DELIMITER
 
+    const val COLUMN_SEPARATOR = SQLUtilsCommons.COLUMN_SEPARATOR
+
     private const val UNDERSCORE = "_"
 
     @JvmStatic

src/main/java/org/mtransit/parser/gtfs/data/GSpec.java

@@ -4,7 +4,6 @@
 import org.jetbrains.annotations.Nullable;
 import org.mtransit.commons.CollectionUtils;
 import org.mtransit.commons.gtfs.data.CalendarDate;
-import org.mtransit.parser.Constants;
 import org.mtransit.parser.DefaultAgencyTools;
 import org.mtransit.parser.FileUtils;
 import org.mtransit.parser.MTLog;
@@ -532,15 +531,15 @@ private void addTripStops(@NotNull GTripStop gTripStop) {
 	@Override
 	public String toString() {
 		return GSpec.class.getSimpleName() + '[' + //
-				AGENCIES + readAgenciesCount() + Constants.COLUMN_SEPARATOR + //
-				CALENDARS + readCalendarsCount() + Constants.COLUMN_SEPARATOR + //
-				CALENDAR_DATES + readCalendarDatesCount() + Constants.COLUMN_SEPARATOR + //
-				ROUTES + readRoutesCount() + Constants.COLUMN_SEPARATOR + //
-				TRIPS + readTripsCount() + Constants.COLUMN_SEPARATOR + //
-				STOPS + readStopsCount() + Constants.COLUMN_SEPARATOR + //
-				STOP_TIMES + readStopTimesCount() + Constants.COLUMN_SEPARATOR + //
-				FREQUENCIES + readFrequenciesCount() + Constants.COLUMN_SEPARATOR + //
-				TRIP_STOPS + readTripStopsCount() + Constants.COLUMN_SEPARATOR + //
+				AGENCIES + readAgenciesCount() + "," + //
+				CALENDARS + readCalendarsCount() + "," + //
+				CALENDAR_DATES + readCalendarDatesCount() + "," + //
+				ROUTES + readRoutesCount() + "," + //
+				TRIPS + readTripsCount() + "," + //
+				STOPS + readStopsCount() + "," + //
+				STOP_TIMES + readStopTimesCount() + "," + //
+				FREQUENCIES + readFrequenciesCount() + "," + //
+				TRIP_STOPS + readTripStopsCount() + "," + //
 				']';
 	}
 

src/main/java/org/mtransit/parser/mt/MGenerator.java

@@ -36,6 +36,8 @@
 import org.mtransit.parser.mt.data.MStop;
 import org.mtransit.parser.mt.data.MDirection;
 import org.mtransit.parser.mt.data.MDirectionStop;
+import org.mtransit.parser.mt.data.MString;
+import org.mtransit.parser.mt.data.MStrings;
 
 import java.io.BufferedWriter;
 import java.io.File;
@@ -176,7 +178,8 @@ public static MSpec generateMSpec(@NotNull GSpec gtfs, @NotNull GAgencyTools age
 		MTLog.log("- Trips: %d", mTripsList.size());
 		MTLog.log("- Trip stops: %d", mDirectionStopsList.size());
 		MTLog.log("- Stops: %d", mStopsList.size());
-		MTLog.log("- Service Ids: %d", MServiceIds.getAll().size());
+		MTLog.log("- Service Ids: %d", MServiceIds.count());
+		MTLog.log("- Strings: %d", MStrings.count());
 		MTLog.log("- Service Dates: %d", mServiceDatesList.size());
 		MTLog.log("- Route with Frequencies: %d", mRouteFrequencies.size());
 		MTLog.log("- First timestamp: %s", MTLog.formatDateTime(firstTimestamp));
@@ -201,6 +204,7 @@ private static void logMerging(@NotNull String msg, long routeId) {
 		MTLog.logDebug("%s: Generating routes, trips, trip stops & stops objects... (merging %s)", routeId, msg);
 	}
 
+	private static final String GTFS_STRINGS = "gtfs_strings";
 	private static final String GTFS_SCHEDULE = "gtfs_schedule";
 	private static final String GTFS_SCHEDULE_SERVICE_DATES = GTFS_SCHEDULE + "_service_dates"; // DB
 	private static final String GTFS_SCHEDULE_SERVICE_IDS = GTFS_SCHEDULE + "_service_ids"; // DB
@@ -282,6 +286,8 @@ public static void dumpFiles(@NotNull GAgencyTools gAgencyTools,
 		dumpFrequencyRoutes(gAgencyTools, mSpec, fileBase, deleteAll, rawDirF);
 		// SERVICE IDS
 		dumpServiceIds(mSpec, fileBase, deleteAll, dataDirF, rawDirF, dbConnection); // AFTER SCHEDULE STOPS & FREQUENCY ROUTES
+		// STRINGS
+		dumpStrings(mSpec, fileBase, deleteAll, dataDirF, rawDirF, dbConnection); // AFTER ALL OTHER TABLE W/ STRINGS
 		if (deleteAll) {
 			dumpValues(rawDirF, fileBase, null, null, null, null, null, -1, -1, null, true);
 		} else {
@@ -568,6 +574,53 @@ private static void dumpServiceIds(
 		}
 	}
 
+	private static void dumpStrings(
+			@Nullable MSpec mSpec,
+			@NotNull String fileBase,
+			boolean deleteAll,
+			@NotNull File dataDirF,
+			@NotNull File rawDirF,
+			@Nullable Connection dbConnection) {
+		if (!FeatureFlags.F_EXPORT_STRINGS) return;
+		if (!deleteAll
+				&& (mSpec == null || !mSpec.isValid() || (F_PRE_FILLED_DB && dbConnection == null))) {
+			throw new MTLog.Fatal("Generated data invalid (agencies: %s)!", mSpec);
+		}
+		if (F_PRE_FILLED_DB) {
+			FileUtils.deleteIfExist(new File(rawDirF, fileBase + GTFS_STRINGS)); // migration from src/main/res/raw to data
+		}
+		File file = new File(F_PRE_FILLED_DB ? dataDirF : rawDirF, fileBase + GTFS_STRINGS);
+		FileUtils.deleteIfExist(file); // delete previous
+		try (BufferedWriter ow = new BufferedWriter(new FileWriter(file))) {
+			if (!deleteAll) {
+				MTLog.logPOINT(); // LOG
+				Statement dbStatement = null;
+				String sqlInsert = null;
+				if (F_PRE_FILLED_DB) {
+					SQLUtils.setAutoCommit(dbConnection, false); // START TRANSACTION
+					dbStatement = dbConnection.createStatement();
+					sqlInsert = GTFSCommons.getT_STRINGS_SQL_INSERT();
+				}
+				for (MString mString : MStrings.getAll()) {
+					final String stringInsert = mString.toFile();
+					if (F_PRE_FILLED_DB) {
+						SQLUtils.executeUpdate(
+								dbStatement,
+								String.format(sqlInsert, stringInsert)
+						);
+					}
+					ow.write(stringInsert);
+					ow.write(Constants.NEW_LINE);
+				}
+				if (F_PRE_FILLED_DB) {
+					SQLUtils.setAutoCommit(dbConnection, true); // END TRANSACTION == commit()
+				}
+			}
+		} catch (Exception ioe) {
+			throw new MTLog.Fatal(ioe, "I/O Error while writing strings file!");
+		}
+	}
+
 	@NotNull
 	private static Pair<Integer, Integer> dumpScheduleServiceDates(@NotNull GAgencyTools gAgencyTools,
 																   @Nullable MSpec mSpec,
@@ -698,7 +751,7 @@ private static void dumpScheduleStops(@NotNull GAgencyTools gAgencyTools,
 							MSchedule lastSchedule = null;
 							for (MSchedule mSchedule : mStopSchedules) {
 								if (mSchedule.isSameServiceAndDirection(lastSchedule)) {
-									ow.write(Constants.COLUMN_SEPARATOR);
+									ow.write(SQLUtils.COLUMN_SEPARATOR);
 									ow.write(mSchedule.toFileSameServiceIdAndDirectionId(lastSchedule));
 								} else {
 									if (!empty) {

src/main/java/org/mtransit/parser/mt/MReader.kt

@@ -14,6 +14,7 @@ import org.mtransit.parser.Pair
 import org.mtransit.parser.gtfs.data.GFieldTypes
 import org.mtransit.parser.mt.data.MServiceDate
 import org.mtransit.parser.mt.data.MServiceId
+import org.mtransit.parser.mt.data.MString
 import java.io.File
 import java.util.TimeZone
 
@@ -164,4 +165,27 @@ object MReader {
     }
 
     // endregion
+
+    // region strings
+
+    private const val GTFS_STRINGS = "gtfs_strings"
+
+    @JvmStatic
+    fun loadStrings(fileBase: String) = try {
+        File(getResDirName(fileBase) + "/$RAW/${fileBase}$GTFS_STRINGS")
+            .takeIf { it.exists() }
+            ?.readLines()
+            ?.mapNotNull { line ->
+                MString.fromFileLine(line)
+            }
+            ?: run {
+                MTLog.log("File not found '${"/$RAW/${fileBase}$GTFS_STRINGS"}'!")
+                null
+            }
+    } catch (e: Exception) {
+        MTLog.logNonFatal(e, "Error while reading '$fileBase' strings!")
+        null
+    }
+
+    // endregion
 }

src/main/java/org/mtransit/parser/mt/data/MDirection.kt

@@ -1,14 +1,14 @@
 package org.mtransit.parser.mt.data
 
-import org.mtransit.parser.Constants
+import org.mtransit.commons.sql.SQLUtils
 import org.mtransit.parser.MTLog
 import org.mtransit.parser.db.SQLUtils.quotesEscape
 
 data class MDirection(
     val routeId: Long,
     var headsignId: Int = 0, // >= 0 (almost = direction ID)
     var headsignType: Int = HEADSIGN_TYPE_STRING, // 0=string, 1=direction, 2=inbound, 3=stopId, 4=descent-only
-    var headsignValue: String = Constants.EMPTY,
+    var headsignValue: String = HEADSIGN_DEFAULT_VALUE,
 ) : Comparable<MDirection> {
 
     constructor(
@@ -17,7 +17,7 @@ data class MDirection(
         routeId = routeId,
         headsignId = 0,
         headsignType = HEADSIGN_TYPE_STRING,
-        headsignValue = Constants.EMPTY,
+        headsignValue = HEADSIGN_DEFAULT_VALUE,
     )
 
     constructor(
@@ -115,7 +115,7 @@ data class MDirection(
     @Suppress("unused")
     fun setHeadsignDescentOnly(): MDirection {
         headsignType = HEADSIGN_TYPE_NO_PICKUP
-        headsignValue = Constants.EMPTY // null;
+        headsignValue = HEADSIGN_DEFAULT_VALUE // null
         headsignId = 0
         _id = -1 // reset
         return this
@@ -177,9 +177,9 @@ data class MDirection(
     fun toFile() = listOf(
         id.toString(), // ID
         headsignType.toString(), // HEADSIGN TYPE
-        headsignValue.quotesEscape(), // HEADSIGN STRING
+        headsignValue.toStringIds().quotesEscape(), // HEADSIGN STRING
         routeId.toString(), // ROUTE ID
-    ).joinToString(Constants.COLUMN_SEPARATOR_)
+    ).joinToString(SQLUtils.COLUMN_SEPARATOR)
 
     override fun compareTo(other: MDirection): Int {
         // sort by direction's route id => direction id
@@ -192,6 +192,8 @@ data class MDirection(
 
     companion object {
 
+        const val HEADSIGN_DEFAULT_VALUE = ""
+
         const val HEADSIGN_TYPE_STRING = 0
         const val HEADSIGN_TYPE_DIRECTION = 1
         const val HEADSIGN_TYPE_INBOUND = 2

src/main/java/org/mtransit/parser/mt/data/MDirectionStop.kt

@@ -1,6 +1,6 @@
 package org.mtransit.parser.mt.data
 
-import org.mtransit.parser.Constants
+import org.mtransit.commons.sql.SQLUtils
 
 data class MDirectionStop(
     val directionId: Long,
@@ -39,7 +39,7 @@ data class MDirectionStop(
         stopId.toString(), // STOP ID
         stopSequence.toString(), // STOP SEQUENCE
         (if (isNoPickup) 1 else 0).toString(), // DROP OFF ONLY
-    ).joinToString(Constants.COLUMN_SEPARATOR_)
+    ).joinToString(SQLUtils.COLUMN_SEPARATOR)
 
     override fun compareTo(other: MDirectionStop): Int {
         // sort by direction_id => stop_sequence

src/main/java/org/mtransit/parser/mt/data/MFrequency.kt

@@ -2,7 +2,7 @@ package org.mtransit.parser.mt.data
 
 import androidx.annotation.Discouraged
 import org.mtransit.commons.FeatureFlags
-import org.mtransit.parser.Constants
+import org.mtransit.commons.sql.SQLUtils
 import org.mtransit.parser.db.SQLUtils.quotesEscape
 import org.mtransit.parser.gtfs.GAgencyTools
 import org.mtransit.parser.gtfs.data.GIDs
@@ -34,7 +34,7 @@ data class MFrequency(
         add(startTime.toString())
         add(endTime.toString())
         add(headwayInSec.toString())
-    }.joinToString(Constants.COLUMN_SEPARATOR_)
+    }.joinToString(SQLUtils.COLUMN_SEPARATOR)
 
     override fun compareTo(other: MFrequency?): Int {
         return when {

src/main/java/org/mtransit/parser/mt/data/MRoute.kt

@@ -1,7 +1,7 @@
 package org.mtransit.parser.mt.data
 
 import org.mtransit.commons.GTFSCommons
-import org.mtransit.parser.Constants
+import org.mtransit.commons.sql.SQLUtils
 import org.mtransit.parser.db.SQLUtils.quotes
 import org.mtransit.parser.db.SQLUtils.quotesEscape
 import org.mtransit.parser.gtfs.GAgencyTools
@@ -33,16 +33,16 @@ data class MRoute(
         type,
     )
 
-    val shortNameOrDefault: String = shortName ?: Constants.EMPTY
+    val shortNameOrDefault: String = shortName.orEmpty()
 
     fun toFile() = buildList {
         add(id.toString()) // ID
-        add((shortName ?: Constants.EMPTY).quotesEscape()) // short name
-        add(longName.quotesEscape()) // long name
-        add((color?.uppercase() ?: Constants.EMPTY).quotes()) // color
+        add(shortName.orEmpty().toStringIds().quotesEscape()) // short name
+        add(longName.toStringIds().quotesEscape()) // long name
+        add((color?.uppercase().orEmpty()).quotes()) // color
         add(originalIdHash.toString()) // original ID hash
         add(type.toString())
-    }.joinToString(Constants.COLUMN_SEPARATOR_)
+    }.joinToString(SQLUtils.COLUMN_SEPARATOR)
 
     override fun compareTo(other: MRoute): Int {
         return id.compareTo(other.id)