diff --git a/src/main/java/org/mtransit/parser/Constants.java b/src/main/java/org/mtransit/parser/Constants.java index 60038b7..8d2b39f 100644 --- a/src/main/java/org/mtransit/parser/Constants.java +++ b/src/main/java/org/mtransit/parser/Constants.java @@ -22,12 +22,6 @@ public final class Constants { public static final char SPACE = ' '; - public static final char COLUMN_SEPARATOR = ','; - - public static final String COLUMN_SEPARATOR_ = ","; - - public static final char STRING_DELIMITER = '\''; - public static final String EMPTY = ""; public static final String SPACE_ = " "; diff --git a/src/main/java/org/mtransit/parser/DefaultAgencyTools.java b/src/main/java/org/mtransit/parser/DefaultAgencyTools.java index 0a2b803..9f27d13 100644 --- a/src/main/java/org/mtransit/parser/DefaultAgencyTools.java +++ b/src/main/java/org/mtransit/parser/DefaultAgencyTools.java @@ -39,6 +39,8 @@ import org.mtransit.parser.mt.data.MServiceIds; import org.mtransit.parser.mt.data.MSpec; import org.mtransit.parser.mt.data.MDirection; +import org.mtransit.parser.mt.data.MString; +import org.mtransit.parser.mt.data.MStrings; import java.text.SimpleDateFormat; import java.util.ArrayList; @@ -154,7 +156,9 @@ public void start(@NotNull String[] args) { MTLog.logDebug("Args [%d]: %s.", args.length, Arrays.asList(args)); final List lastServiceDates = MReader.loadServiceDates(args[2]); final List lastServiceIds = MReader.loadServiceIds(args[2]); + final List lastStrings = MReader.loadStrings(args[2]); MServiceIds.addAll(lastServiceIds); + MStrings.addAll(lastStrings); this.serviceIdInts = extractUsefulServiceIdInts(args, this, true, lastServiceDates); final String inputUrl = args.length >= 5 ? args[4] : null; if (excludingAll()) { diff --git a/src/main/java/org/mtransit/parser/db/DumpDbUtils.kt b/src/main/java/org/mtransit/parser/db/DumpDbUtils.kt index c5857ef..acac9ca 100644 --- a/src/main/java/org/mtransit/parser/db/DumpDbUtils.kt +++ b/src/main/java/org/mtransit/parser/db/DumpDbUtils.kt @@ -27,6 +27,7 @@ object DumpDbUtils { connection.createStatement().use { statement -> SQLUtils.execute(statement, "PRAGMA auto_vacuum = NONE") // DROP IF EXIST + SQLUtils.executeUpdate(statement, GTFSCommons.T_STRINGS_SQL_DROP) SQLUtils.executeUpdate(statement, GTFSCommons.T_DIRECTION_STOPS_SQL_DROP) SQLUtils.executeUpdate(statement, GTFSCommons.T_STOP_SQL_DROP) SQLUtils.executeUpdate(statement, GTFSCommons.T_DIRECTION_SQL_DROP) @@ -44,6 +45,7 @@ object DumpDbUtils { SQLUtils.executeUpdate(statement, GTFSCommons.T_SERVICE_IDS_SQL_CREATE) } SQLUtils.executeUpdate(statement, GTFSCommons.T_SERVICE_DATES_SQL_CREATE) + SQLUtils.executeUpdate(statement, GTFSCommons.T_STRINGS_SQL_CREATE) } } } \ No newline at end of file diff --git a/src/main/java/org/mtransit/parser/db/SQLUtils.kt b/src/main/java/org/mtransit/parser/db/SQLUtils.kt index 0eb587d..9176d93 100644 --- a/src/main/java/org/mtransit/parser/db/SQLUtils.kt +++ b/src/main/java/org/mtransit/parser/db/SQLUtils.kt @@ -21,6 +21,8 @@ object SQLUtils { private const val QUOTE_ = '\'' private const val QUOTE = SQLUtilsCommons.STRING_DELIMITER + const val COLUMN_SEPARATOR = SQLUtilsCommons.COLUMN_SEPARATOR + private const val UNDERSCORE = "_" @JvmStatic diff --git a/src/main/java/org/mtransit/parser/gtfs/data/GSpec.java b/src/main/java/org/mtransit/parser/gtfs/data/GSpec.java index 9f29a86..6b8968b 100644 --- a/src/main/java/org/mtransit/parser/gtfs/data/GSpec.java +++ b/src/main/java/org/mtransit/parser/gtfs/data/GSpec.java @@ -4,7 +4,6 @@ import org.jetbrains.annotations.Nullable; import org.mtransit.commons.CollectionUtils; import org.mtransit.commons.gtfs.data.CalendarDate; -import org.mtransit.parser.Constants; import org.mtransit.parser.DefaultAgencyTools; import org.mtransit.parser.FileUtils; import org.mtransit.parser.MTLog; @@ -532,15 +531,15 @@ private void addTripStops(@NotNull GTripStop gTripStop) { @Override public String toString() { return GSpec.class.getSimpleName() + '[' + // - AGENCIES + readAgenciesCount() + Constants.COLUMN_SEPARATOR + // - CALENDARS + readCalendarsCount() + Constants.COLUMN_SEPARATOR + // - CALENDAR_DATES + readCalendarDatesCount() + Constants.COLUMN_SEPARATOR + // - ROUTES + readRoutesCount() + Constants.COLUMN_SEPARATOR + // - TRIPS + readTripsCount() + Constants.COLUMN_SEPARATOR + // - STOPS + readStopsCount() + Constants.COLUMN_SEPARATOR + // - STOP_TIMES + readStopTimesCount() + Constants.COLUMN_SEPARATOR + // - FREQUENCIES + readFrequenciesCount() + Constants.COLUMN_SEPARATOR + // - TRIP_STOPS + readTripStopsCount() + Constants.COLUMN_SEPARATOR + // + AGENCIES + readAgenciesCount() + "," + // + CALENDARS + readCalendarsCount() + "," + // + CALENDAR_DATES + readCalendarDatesCount() + "," + // + ROUTES + readRoutesCount() + "," + // + TRIPS + readTripsCount() + "," + // + STOPS + readStopsCount() + "," + // + STOP_TIMES + readStopTimesCount() + "," + // + FREQUENCIES + readFrequenciesCount() + "," + // + TRIP_STOPS + readTripStopsCount() + "," + // ']'; } diff --git a/src/main/java/org/mtransit/parser/mt/MGenerator.java b/src/main/java/org/mtransit/parser/mt/MGenerator.java index 9c3e570..cab2d31 100644 --- a/src/main/java/org/mtransit/parser/mt/MGenerator.java +++ b/src/main/java/org/mtransit/parser/mt/MGenerator.java @@ -36,6 +36,8 @@ import org.mtransit.parser.mt.data.MStop; import org.mtransit.parser.mt.data.MDirection; import org.mtransit.parser.mt.data.MDirectionStop; +import org.mtransit.parser.mt.data.MString; +import org.mtransit.parser.mt.data.MStrings; import java.io.BufferedWriter; import java.io.File; @@ -176,7 +178,8 @@ public static MSpec generateMSpec(@NotNull GSpec gtfs, @NotNull GAgencyTools age MTLog.log("- Trips: %d", mTripsList.size()); MTLog.log("- Trip stops: %d", mDirectionStopsList.size()); MTLog.log("- Stops: %d", mStopsList.size()); - MTLog.log("- Service Ids: %d", MServiceIds.getAll().size()); + MTLog.log("- Service Ids: %d", MServiceIds.count()); + MTLog.log("- Strings: %d", MStrings.count()); MTLog.log("- Service Dates: %d", mServiceDatesList.size()); MTLog.log("- Route with Frequencies: %d", mRouteFrequencies.size()); MTLog.log("- First timestamp: %s", MTLog.formatDateTime(firstTimestamp)); @@ -201,6 +204,7 @@ private static void logMerging(@NotNull String msg, long routeId) { MTLog.logDebug("%s: Generating routes, trips, trip stops & stops objects... (merging %s)", routeId, msg); } + private static final String GTFS_STRINGS = "gtfs_strings"; private static final String GTFS_SCHEDULE = "gtfs_schedule"; private static final String GTFS_SCHEDULE_SERVICE_DATES = GTFS_SCHEDULE + "_service_dates"; // DB private static final String GTFS_SCHEDULE_SERVICE_IDS = GTFS_SCHEDULE + "_service_ids"; // DB @@ -282,6 +286,8 @@ public static void dumpFiles(@NotNull GAgencyTools gAgencyTools, dumpFrequencyRoutes(gAgencyTools, mSpec, fileBase, deleteAll, rawDirF); // SERVICE IDS dumpServiceIds(mSpec, fileBase, deleteAll, dataDirF, rawDirF, dbConnection); // AFTER SCHEDULE STOPS & FREQUENCY ROUTES + // STRINGS + dumpStrings(mSpec, fileBase, deleteAll, dataDirF, rawDirF, dbConnection); // AFTER ALL OTHER TABLE W/ STRINGS if (deleteAll) { dumpValues(rawDirF, fileBase, null, null, null, null, null, -1, -1, null, true); } else { @@ -568,6 +574,53 @@ private static void dumpServiceIds( } } + private static void dumpStrings( + @Nullable MSpec mSpec, + @NotNull String fileBase, + boolean deleteAll, + @NotNull File dataDirF, + @NotNull File rawDirF, + @Nullable Connection dbConnection) { + if (!FeatureFlags.F_EXPORT_STRINGS) return; + if (!deleteAll + && (mSpec == null || !mSpec.isValid() || (F_PRE_FILLED_DB && dbConnection == null))) { + throw new MTLog.Fatal("Generated data invalid (agencies: %s)!", mSpec); + } + if (F_PRE_FILLED_DB) { + FileUtils.deleteIfExist(new File(rawDirF, fileBase + GTFS_STRINGS)); // migration from src/main/res/raw to data + } + File file = new File(F_PRE_FILLED_DB ? dataDirF : rawDirF, fileBase + GTFS_STRINGS); + FileUtils.deleteIfExist(file); // delete previous + try (BufferedWriter ow = new BufferedWriter(new FileWriter(file))) { + if (!deleteAll) { + MTLog.logPOINT(); // LOG + Statement dbStatement = null; + String sqlInsert = null; + if (F_PRE_FILLED_DB) { + SQLUtils.setAutoCommit(dbConnection, false); // START TRANSACTION + dbStatement = dbConnection.createStatement(); + sqlInsert = GTFSCommons.getT_STRINGS_SQL_INSERT(); + } + for (MString mString : MStrings.getAll()) { + final String stringInsert = mString.toFile(); + if (F_PRE_FILLED_DB) { + SQLUtils.executeUpdate( + dbStatement, + String.format(sqlInsert, stringInsert) + ); + } + ow.write(stringInsert); + ow.write(Constants.NEW_LINE); + } + if (F_PRE_FILLED_DB) { + SQLUtils.setAutoCommit(dbConnection, true); // END TRANSACTION == commit() + } + } + } catch (Exception ioe) { + throw new MTLog.Fatal(ioe, "I/O Error while writing strings file!"); + } + } + @NotNull private static Pair dumpScheduleServiceDates(@NotNull GAgencyTools gAgencyTools, @Nullable MSpec mSpec, @@ -698,7 +751,7 @@ private static void dumpScheduleStops(@NotNull GAgencyTools gAgencyTools, MSchedule lastSchedule = null; for (MSchedule mSchedule : mStopSchedules) { if (mSchedule.isSameServiceAndDirection(lastSchedule)) { - ow.write(Constants.COLUMN_SEPARATOR); + ow.write(SQLUtils.COLUMN_SEPARATOR); ow.write(mSchedule.toFileSameServiceIdAndDirectionId(lastSchedule)); } else { if (!empty) { diff --git a/src/main/java/org/mtransit/parser/mt/MReader.kt b/src/main/java/org/mtransit/parser/mt/MReader.kt index daecf66..0d9f6fb 100644 --- a/src/main/java/org/mtransit/parser/mt/MReader.kt +++ b/src/main/java/org/mtransit/parser/mt/MReader.kt @@ -14,6 +14,7 @@ import org.mtransit.parser.Pair import org.mtransit.parser.gtfs.data.GFieldTypes import org.mtransit.parser.mt.data.MServiceDate import org.mtransit.parser.mt.data.MServiceId +import org.mtransit.parser.mt.data.MString import java.io.File import java.util.TimeZone @@ -164,4 +165,27 @@ object MReader { } // endregion + + // region strings + + private const val GTFS_STRINGS = "gtfs_strings" + + @JvmStatic + fun loadStrings(fileBase: String) = try { + File(getResDirName(fileBase) + "/$RAW/${fileBase}$GTFS_STRINGS") + .takeIf { it.exists() } + ?.readLines() + ?.mapNotNull { line -> + MString.fromFileLine(line) + } + ?: run { + MTLog.log("File not found '${"/$RAW/${fileBase}$GTFS_STRINGS"}'!") + null + } + } catch (e: Exception) { + MTLog.logNonFatal(e, "Error while reading '$fileBase' strings!") + null + } + + // endregion } \ No newline at end of file diff --git a/src/main/java/org/mtransit/parser/mt/data/MDirection.kt b/src/main/java/org/mtransit/parser/mt/data/MDirection.kt index efaffec..be6f4e0 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MDirection.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MDirection.kt @@ -1,6 +1,6 @@ package org.mtransit.parser.mt.data -import org.mtransit.parser.Constants +import org.mtransit.commons.sql.SQLUtils import org.mtransit.parser.MTLog import org.mtransit.parser.db.SQLUtils.quotesEscape @@ -8,7 +8,7 @@ data class MDirection( val routeId: Long, var headsignId: Int = 0, // >= 0 (almost = direction ID) var headsignType: Int = HEADSIGN_TYPE_STRING, // 0=string, 1=direction, 2=inbound, 3=stopId, 4=descent-only - var headsignValue: String = Constants.EMPTY, + var headsignValue: String = HEADSIGN_DEFAULT_VALUE, ) : Comparable { constructor( @@ -17,7 +17,7 @@ data class MDirection( routeId = routeId, headsignId = 0, headsignType = HEADSIGN_TYPE_STRING, - headsignValue = Constants.EMPTY, + headsignValue = HEADSIGN_DEFAULT_VALUE, ) constructor( @@ -115,7 +115,7 @@ data class MDirection( @Suppress("unused") fun setHeadsignDescentOnly(): MDirection { headsignType = HEADSIGN_TYPE_NO_PICKUP - headsignValue = Constants.EMPTY // null; + headsignValue = HEADSIGN_DEFAULT_VALUE // null headsignId = 0 _id = -1 // reset return this @@ -177,9 +177,9 @@ data class MDirection( fun toFile() = listOf( id.toString(), // ID headsignType.toString(), // HEADSIGN TYPE - headsignValue.quotesEscape(), // HEADSIGN STRING + headsignValue.toStringIds().quotesEscape(), // HEADSIGN STRING routeId.toString(), // ROUTE ID - ).joinToString(Constants.COLUMN_SEPARATOR_) + ).joinToString(SQLUtils.COLUMN_SEPARATOR) override fun compareTo(other: MDirection): Int { // sort by direction's route id => direction id @@ -192,6 +192,8 @@ data class MDirection( companion object { + const val HEADSIGN_DEFAULT_VALUE = "" + const val HEADSIGN_TYPE_STRING = 0 const val HEADSIGN_TYPE_DIRECTION = 1 const val HEADSIGN_TYPE_INBOUND = 2 diff --git a/src/main/java/org/mtransit/parser/mt/data/MDirectionStop.kt b/src/main/java/org/mtransit/parser/mt/data/MDirectionStop.kt index cc3f7b3..f5138a1 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MDirectionStop.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MDirectionStop.kt @@ -1,6 +1,6 @@ package org.mtransit.parser.mt.data -import org.mtransit.parser.Constants +import org.mtransit.commons.sql.SQLUtils data class MDirectionStop( val directionId: Long, @@ -39,7 +39,7 @@ data class MDirectionStop( stopId.toString(), // STOP ID stopSequence.toString(), // STOP SEQUENCE (if (isNoPickup) 1 else 0).toString(), // DROP OFF ONLY - ).joinToString(Constants.COLUMN_SEPARATOR_) + ).joinToString(SQLUtils.COLUMN_SEPARATOR) override fun compareTo(other: MDirectionStop): Int { // sort by direction_id => stop_sequence diff --git a/src/main/java/org/mtransit/parser/mt/data/MFrequency.kt b/src/main/java/org/mtransit/parser/mt/data/MFrequency.kt index c63de4d..01d4637 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MFrequency.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MFrequency.kt @@ -2,7 +2,7 @@ package org.mtransit.parser.mt.data import androidx.annotation.Discouraged import org.mtransit.commons.FeatureFlags -import org.mtransit.parser.Constants +import org.mtransit.commons.sql.SQLUtils import org.mtransit.parser.db.SQLUtils.quotesEscape import org.mtransit.parser.gtfs.GAgencyTools import org.mtransit.parser.gtfs.data.GIDs @@ -34,7 +34,7 @@ data class MFrequency( add(startTime.toString()) add(endTime.toString()) add(headwayInSec.toString()) - }.joinToString(Constants.COLUMN_SEPARATOR_) + }.joinToString(SQLUtils.COLUMN_SEPARATOR) override fun compareTo(other: MFrequency?): Int { return when { diff --git a/src/main/java/org/mtransit/parser/mt/data/MRoute.kt b/src/main/java/org/mtransit/parser/mt/data/MRoute.kt index 3eeda6c..00bdc7b 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MRoute.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MRoute.kt @@ -1,7 +1,7 @@ package org.mtransit.parser.mt.data import org.mtransit.commons.GTFSCommons -import org.mtransit.parser.Constants +import org.mtransit.commons.sql.SQLUtils import org.mtransit.parser.db.SQLUtils.quotes import org.mtransit.parser.db.SQLUtils.quotesEscape import org.mtransit.parser.gtfs.GAgencyTools @@ -33,16 +33,16 @@ data class MRoute( type, ) - val shortNameOrDefault: String = shortName ?: Constants.EMPTY + val shortNameOrDefault: String = shortName.orEmpty() fun toFile() = buildList { add(id.toString()) // ID - add((shortName ?: Constants.EMPTY).quotesEscape()) // short name - add(longName.quotesEscape()) // long name - add((color?.uppercase() ?: Constants.EMPTY).quotes()) // color + add(shortName.orEmpty().toStringIds().quotesEscape()) // short name + add(longName.toStringIds().quotesEscape()) // long name + add((color?.uppercase().orEmpty()).quotes()) // color add(originalIdHash.toString()) // original ID hash add(type.toString()) - }.joinToString(Constants.COLUMN_SEPARATOR_) + }.joinToString(SQLUtils.COLUMN_SEPARATOR) override fun compareTo(other: MRoute): Int { return id.compareTo(other.id) diff --git a/src/main/java/org/mtransit/parser/mt/data/MSchedule.kt b/src/main/java/org/mtransit/parser/mt/data/MSchedule.kt index a5956f3..85ad743 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MSchedule.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MSchedule.kt @@ -2,7 +2,7 @@ package org.mtransit.parser.mt.data import androidx.annotation.Discouraged import org.mtransit.commons.FeatureFlags -import org.mtransit.parser.Constants +import org.mtransit.commons.sql.SQLUtils import org.mtransit.parser.DefaultAgencyTools import org.mtransit.parser.MTLog import org.mtransit.parser.Pair @@ -98,11 +98,13 @@ data class MSchedule( } fun toFileNewServiceIdAndDirectionId(agencyTools: GAgencyTools) = buildList { - if (FeatureFlags.F_EXPORT_SERVICE_ID_INTS) { - add(MServiceIds.getInt(agencyTools.cleanServiceId(_serviceId))) - } else { - add(agencyTools.cleanServiceId(_serviceId).quotesEscape()) - } + add( + if (FeatureFlags.F_EXPORT_SERVICE_ID_INTS) { + MServiceIds.getInt(agencyTools.cleanServiceId(_serviceId)) + } else { + agencyTools.cleanServiceId(_serviceId).quotesEscape() + } + ) // no route ID, just for file split add(directionId.toString()) add(departure.toString()) @@ -111,13 +113,13 @@ data class MSchedule( if (arrivalBeforeDeparture > 0) { // TODO ? } - add(arrivalBeforeDeparture.takeIf { it > 0 }?.toString() ?: Constants.EMPTY) // arrival before departure + add(arrivalBeforeDeparture.takeIf { it > 0 }?.toString().orEmpty()) // arrival before departure add(_tripId.quotesEscape()) } - add(headsignType.takeIf { it >= 0 }?.toString() ?: Constants.EMPTY) - add((headsignValue ?: Constants.EMPTY).quotesEscape()) + add(headsignType.takeIf { it >= 0 }?.toString().orEmpty()) + add(headsignValue.orEmpty().toStringIds().quotesEscape()) add(accessible.toString()) - }.joinToString(Constants.COLUMN_SEPARATOR_) + }.joinToString(SQLUtils.COLUMN_SEPARATOR) fun toFileSameServiceIdAndDirectionId(lastSchedule: MSchedule?) = buildList { add((departure - (lastSchedule?.departure ?: 0)).toString()) @@ -126,18 +128,18 @@ data class MSchedule( if (arrivalBeforeDeparture > 0) { // TODO ? } - add(arrivalBeforeDeparture.takeIf { it > 0 }?.toString() ?: Constants.EMPTY) // arrival before departure + add(arrivalBeforeDeparture.takeIf { it > 0 }?.toString().orEmpty()) // arrival before departure add(_tripId.quotesEscape()) } if (headsignType == MDirection.HEADSIGN_TYPE_NO_PICKUP) { add(MDirection.HEADSIGN_TYPE_NO_PICKUP.toString()) - add(Constants.EMPTY.quotes()) + add(MDirection.HEADSIGN_DEFAULT_VALUE.quotes()) } else { - add(headsignType.takeIf { it >= 0 }?.toString() ?: Constants.EMPTY) - add((headsignValue ?: Constants.EMPTY).quotesEscape()) + add(headsignType.takeIf { it >= 0 }?.toString().orEmpty()) + add(headsignValue.orEmpty().toStringIds().quotesEscape()) } add(accessible.toString()) - }.joinToString(Constants.COLUMN_SEPARATOR_) + }.joinToString(SQLUtils.COLUMN_SEPARATOR) fun isSameServiceAndDirection(lastSchedule: MSchedule?): Boolean { return lastSchedule?.serviceIdInt == serviceIdInt diff --git a/src/main/java/org/mtransit/parser/mt/data/MServiceDate.kt b/src/main/java/org/mtransit/parser/mt/data/MServiceDate.kt index ae6615a..0ca1090 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MServiceDate.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MServiceDate.kt @@ -2,7 +2,7 @@ package org.mtransit.parser.mt.data import androidx.annotation.Discouraged import org.mtransit.commons.FeatureFlags -import org.mtransit.parser.Constants +import org.mtransit.commons.sql.SQLUtils import org.mtransit.parser.db.SQLUtils.quotesEscape import org.mtransit.parser.db.SQLUtils.unquotes import org.mtransit.parser.gtfs.GAgencyTools @@ -47,7 +47,7 @@ data class MServiceDate( } add(calendarDate.toString()) add(exceptionType.toString()) - }.joinToString(Constants.COLUMN_SEPARATOR_) + }.joinToString(SQLUtils.COLUMN_SEPARATOR) @Suppress("unused") fun toStringPlus(): String { @@ -88,7 +88,7 @@ data class MServiceDate( ) fun fromFileLine(line: String) = - line.split(Constants.COLUMN_SEPARATOR) + line.split(SQLUtils.COLUMN_SEPARATOR) .takeIf { it.size == 3 } ?.let { columns -> MServiceDate( diff --git a/src/main/java/org/mtransit/parser/mt/data/MServiceId.kt b/src/main/java/org/mtransit/parser/mt/data/MServiceId.kt index bb34609..f04e077 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MServiceId.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MServiceId.kt @@ -1,9 +1,9 @@ package org.mtransit.parser.mt.data -import org.mtransit.parser.Constants +import org.mtransit.commons.sql.SQLUtils +import org.mtransit.commons.sql.SQLUtils.unquotesUnescape import org.mtransit.parser.MTLog import org.mtransit.parser.db.SQLUtils.quotesEscape -import org.mtransit.parser.db.SQLUtils.unquotes data class MServiceId( val serviceIdInt: Int, @@ -16,7 +16,7 @@ data class MServiceId( fun toFile() = buildList { add(serviceIdInt.toString()) add(serviceId.quotesEscape()) // already agencyTools.cleanServiceId(serviceId) before - }.joinToString(Constants.COLUMN_SEPARATOR_) + }.joinToString(SQLUtils.COLUMN_SEPARATOR) override fun compareTo(other: MServiceId) = compareBy( MServiceId::serviceIdInt, @@ -24,13 +24,13 @@ data class MServiceId( ).compare(this, other) companion object { - fun fromFileLine(line: String): MServiceId? = - line.split(Constants.COLUMN_SEPARATOR) + fun fromFileLine(line: String) = + line.split(SQLUtils.COLUMN_SEPARATOR) .takeIf { it.size == 2 } ?.let { columns -> MServiceId( serviceIdInt = columns[0].toInt(), - serviceId = columns[1].unquotes(), + serviceId = columns[1].unquotesUnescape(), ) } ?: run { diff --git a/src/main/java/org/mtransit/parser/mt/data/MServiceIds.kt b/src/main/java/org/mtransit/parser/mt/data/MServiceIds.kt index 4671e91..5692384 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MServiceIds.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MServiceIds.kt @@ -46,6 +46,9 @@ object MServiceIds { return idToIdInt[serviceId] ?: add(serviceId) } + @JvmStatic + fun count() = idIntToId.size() + @JvmStatic fun getAll() = buildList { idToIdInt.forEach { id, idInt -> diff --git a/src/main/java/org/mtransit/parser/mt/data/MStop.kt b/src/main/java/org/mtransit/parser/mt/data/MStop.kt index 75bb828..0dae53b 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MStop.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MStop.kt @@ -1,7 +1,7 @@ package org.mtransit.parser.mt.data import org.mtransit.commons.GTFSCommons -import org.mtransit.parser.Constants +import org.mtransit.commons.sql.SQLUtils import org.mtransit.parser.db.SQLUtils.quotesEscape import org.mtransit.parser.gtfs.GAgencyTools import org.mtransit.parser.mt.MDataChangedManager @@ -46,12 +46,12 @@ data class MStop( fun toFile() = listOf( id.toString(), // ID code.quotesEscape(), // code - name.quotesEscape(), // name + name.toStringIds().quotesEscape(), // name MDataChangedManager.avoidLatLngChanged(lat), // latitude MDataChangedManager.avoidLatLngChanged(lng), // longitude accessible.toString(), originalIdHash.toString(), // original ID hash - ).joinToString(Constants.COLUMN_SEPARATOR_) + ).joinToString(SQLUtils.COLUMN_SEPARATOR) override fun compareTo(other: MStop): Int { return id - other.id diff --git a/src/main/java/org/mtransit/parser/mt/data/MString.kt b/src/main/java/org/mtransit/parser/mt/data/MString.kt new file mode 100644 index 0000000..242d58a --- /dev/null +++ b/src/main/java/org/mtransit/parser/mt/data/MString.kt @@ -0,0 +1,41 @@ +package org.mtransit.parser.mt.data + +import org.mtransit.commons.sql.SQLUtils +import org.mtransit.commons.sql.SQLUtils.unquotesUnescape +import org.mtransit.parser.MTLog +import org.mtransit.parser.db.SQLUtils.quotesEscape + +data class MString( + val id: Int, + val string: String, +) : Comparable { + + /** + * same order as [org.mtransit.commons.GTFSCommons.T_STRINGS_SQL_INSERT] + */ + fun toFile() = buildList { + add(id.toString()) + add(string.quotesEscape()) + }.joinToString(SQLUtils.COLUMN_SEPARATOR) + + override fun compareTo(other: MString) = compareBy( + MString::id, + MString::string, + ).compare(this, other) + + companion object { + fun fromFileLine(line: String) = + line.split(SQLUtils.COLUMN_SEPARATOR) + .takeIf { it.size == 2 } + ?.let { columns -> + MString( + id = columns[0].toInt(), + string = columns[1].unquotesUnescape(), + ) + } + ?: run { + MTLog.log("Invalid string line: '$line'!") + null + } + } +} diff --git a/src/main/java/org/mtransit/parser/mt/data/MStrings.kt b/src/main/java/org/mtransit/parser/mt/data/MStrings.kt new file mode 100644 index 0000000..394f690 --- /dev/null +++ b/src/main/java/org/mtransit/parser/mt/data/MStrings.kt @@ -0,0 +1,78 @@ +package org.mtransit.parser.mt.data + +import androidx.collection.SparseArrayCompat +import androidx.collection.mutableScatterMapOf +import org.mtransit.commons.FeatureFlags +import org.mtransit.commons.GTFSCommons +import org.mtransit.parser.MTLog + +object MStrings { + + private val idIntToId = SparseArrayCompat() + private val idToIdInt = mutableScatterMapOf() + + private val incrementLock = Any() + private var increment = 0 + + init { + // STATIC STRINGS: (frequently used) + add("/") + add("-") + add("&") + add("@") + } + + @JvmStatic + fun addAll(lastStrings: List?) { + synchronized(incrementLock) { + lastStrings?.forEach { addSynchronized(it) } + } + } + + private fun addSynchronized(string: MString) { + idIntToId.put(string.id, string.string) + idToIdInt[string.string] = string.id + increment = maxOf(increment, string.id) + } + + fun add(string: String): Int { + synchronized(incrementLock) { + increment++ // move to next + return MString(increment, string) + .apply { addSynchronized(this) } + .id + } + } + + @Suppress("unused") + @JvmStatic + fun getString(id: Int) = + idIntToId[id] ?: throw MTLog.Fatal("Unexpected string integer $id!") + + @JvmStatic + fun getInt(string: String): Int = + idToIdInt[string] + ?: synchronized(incrementLock) { idToIdInt[string] ?: add(string) } + + @JvmStatic + fun count() = idIntToId.size() + + @JvmStatic + fun getAll() = buildList { + idToIdInt.forEach { id, idInt -> + add(MString(idInt, id)) + } + }.sorted() + + @JvmStatic + fun convert(strings: String): String { + if (!FeatureFlags.F_EXPORT_STRINGS) return strings + if (strings.isEmpty()) return strings + return strings + .split(GTFSCommons.STRINGS_SEPARATOR) + .map { getInt(it) } + .joinToString(GTFSCommons.STRINGS_SEPARATOR) + } +} + +fun String.toStringIds() = MStrings.convert(this) \ No newline at end of file