feat!(website, backend): Update bulk file submission/revision format to use metadata file

backend/src/main/kotlin/org/loculus/backend/api/SubmissionTypes.kt

@@ -478,3 +478,8 @@ fun FileCategoryFilesMap.getDuplicateFileNames(category: FileCategory): Set<Stri
 
     return nameCounts.filterValues { it > 1 }.keys
 }
+
+fun FileCategoryFilesMap.mergeFileCategories(override: FileCategoryFilesMap?): FileCategoryFilesMap {
+    if (override == null) return this
+    return this + override
+}

backend/src/main/kotlin/org/loculus/backend/controller/SubmissionController.kt

@@ -527,6 +527,11 @@ open class SubmissionController(
         val instanceConfig = backendConfig.getInstanceConfig(organism)
         val hasConsensusSequences = instanceConfig.schema.submissionDataTypes.consensusSequences
         val isMultiSegmented = instanceConfig.referenceGenome.nucleotideSequences.size > 1
+        val fileCategories = if (instanceConfig.schema.submissionDataTypes.files.enabled) {
+            instanceConfig.schema.submissionDataTypes.files.categories
+        } else {
+            emptyList()
+        }
 
         val streamBody = StreamingResponseBody { responseBodyStream ->
             val startTime = System.currentTimeMillis()
@@ -556,6 +561,7 @@ open class SubmissionController(
                             uniqueFastaIdsByEntry,
                             zipOut,
                             isMultiSegmented,
+                            fileCategories,
                         )
                         zipOut.closeEntry()
 

backend/src/main/kotlin/org/loculus/backend/model/SubmitModel.kt

@@ -33,6 +33,9 @@ const val METADATA_ID_HEADER_ALTERNATE_FOR_BACKCOMPAT = "submissionId"
 const val FASTA_IDS_HEADER = "fastaIds"
 const val FASTA_IDS_SEPARATOR = " "
 
+const val FILES_HEADER_PREFIX = "files."
+const val FILES_SEPARATOR = " "
+
 const val ACCESSION_HEADER = "accession"
 private val log = KotlinLogging.logger { }
 
@@ -144,10 +147,20 @@ class SubmitModel(
             )
         }
 
-        submissionParams.files?.let { submittedFiles ->
-            val fileSubmissionIds = submittedFiles.keys
-            validateSubmissionIdSetsForFiles(metadataSubmissionIds, fileSubmissionIds)
-            validateFileGroupOwnership(submittedFiles, submissionParams, uploadId)
+        // File mappings in submissionParams can contain submission Ids not present in the metadata
+        // This is implicitly validated for the file mappings within the metadata column
+        // TODO: This can be removed once file mappings JSON support is removed
+        submissionParams.files?.let { validateSubmissionIdSetsForFiles(metadataSubmissionIds, it.keys) }
+
+        val files = uploadDatabaseService.getFilesForUpload(uploadId)
+        if (files.isNotEmpty()) {
+            submissionIdFilesMappingPreconditionValidator
+                .validateFilenameCharacters(files)
+                .validateFilenamesAreUnique(files)
+                .validateCategoriesMatchSchema(files, submissionParams.organism)
+                .validateMultipartUploads(files)
+                .validateFilesExist(files)
+            validateFileGroupOwnership(files, submissionParams, uploadId)
         }
 
         if (submissionParams is SubmissionParams.OriginalSubmissionParams) {

backend/src/main/kotlin/org/loculus/backend/service/submission/UploadDatabaseService.kt

@@ -16,6 +16,7 @@ import org.loculus.backend.api.Organism
 import org.loculus.backend.api.Status
 import org.loculus.backend.api.SubmissionIdFilesMap
 import org.loculus.backend.api.SubmissionIdMapping
+import org.loculus.backend.api.mergeFileCategories
 import org.loculus.backend.auth.AuthenticatedUser
 import org.loculus.backend.controller.UnprocessableEntityException
 import org.loculus.backend.log.AuditLogger
@@ -81,7 +82,7 @@ class UploadDatabaseService(
                 this[submissionIdColumn] = it.submissionId
                 this[fastaIdsColumn] = it.fastaIds?.toList()
                 this[metadataColumn] = it.metadata
-                this[filesColumn] = files?.get(it.submissionId)
+                this[filesColumn] = files?.get(it.submissionId)?.mergeFileCategories(it.files) ?: it.files
                 this[organismColumn] = submittedOrganism.name
                 this[uploadIdColumn] = uploadId
             }
@@ -120,7 +121,7 @@ class UploadDatabaseService(
                     this[submissionIdColumn] = it.submissionId
                     this[fastaIdsColumn] = it.fastaIds?.toList()
                     this[metadataColumn] = it.metadata
-                    this[filesColumn] = files?.get(it.submissionId)
+                    this[filesColumn] = files?.get(it.submissionId)?.mergeFileCategories(it.files) ?: it.files
                     this[organismColumn] = submittedOrganism.name
                     this[uploadIdColumn] = uploadId
                 }
@@ -166,6 +167,15 @@ class UploadDatabaseService(
         .where { uploadIdColumn eq uploadId }
         .map { it[submissionIdColumn] }
 
+    fun getFilesForUpload(uploadId: String): SubmissionIdFilesMap = MetadataUploadAuxTable
+        .select(
+            submissionIdColumn,
+            filesColumn,
+        )
+        .where { uploadIdColumn eq uploadId }
+        .mapNotNull { row -> row[filesColumn]?.let { row[submissionIdColumn] to it } }
+        .toMap()
+
     fun getFastaIdsForMetadata(uploadId: String): List<List<String>> = MetadataUploadAuxTable
         .select(
             uploadIdColumn,

backend/src/main/kotlin/org/loculus/backend/utils/GetSubmittedDataHelpers.kt

@@ -1,9 +1,13 @@
 package org.loculus.backend.utils
 
+import org.loculus.backend.api.FileIdAndName
 import org.loculus.backend.api.SubmittedDataDownloadEntry
+import org.loculus.backend.config.FileCategory
 import org.loculus.backend.model.ACCESSION_HEADER
 import org.loculus.backend.model.FASTA_IDS_HEADER
 import org.loculus.backend.model.FASTA_IDS_SEPARATOR
+import org.loculus.backend.model.FILES_HEADER_PREFIX
+import org.loculus.backend.model.FILES_SEPARATOR
 import org.loculus.backend.model.FastaId
 import org.loculus.backend.model.METADATA_ID_HEADER
 import kotlin.collections.component1
@@ -20,6 +24,9 @@ data class UniqueFastaIdsForEntry(val uniqueFastaIdByOriginalFastaId: Map<FastaI
     fun getUniqueFastaId(originalFastaId: FastaId): FastaId = uniqueFastaIdByOriginalFastaId.getValue(originalFastaId)
 }
 
+private fun formatFilesCell(files: List<FileIdAndName>?): String =
+    files.orEmpty().joinToString(FILES_SEPARATOR) { "${it.name}:${it.fileId}" }
+
 object GetSubmittedDataHelpers {
 
     fun uniqueFastaIdsByEntry(
@@ -47,22 +54,27 @@ object GetSubmittedDataHelpers {
         fastaIdsByEntry: List<UniqueFastaIdsForEntry>,
         outputStream: java.io.OutputStream,
         isMultiSegmented: Boolean,
+        fileCategories: List<FileCategory>,
     ) {
         val metadataKeys = data.flatMapTo(mutableSetOf()) { it.submittedData.metadata.keys }.sorted()
+        val fileColumnHeaders = fileCategories.map { "$FILES_HEADER_PREFIX${it.name}" }
         val headers = if (isMultiSegmented) {
-            listOf(METADATA_ID_HEADER, ACCESSION_HEADER, FASTA_IDS_HEADER) + metadataKeys
+            listOf(METADATA_ID_HEADER, ACCESSION_HEADER, FASTA_IDS_HEADER) + metadataKeys + fileColumnHeaders
         } else {
-            listOf(METADATA_ID_HEADER, ACCESSION_HEADER) + metadataKeys
+            listOf(METADATA_ID_HEADER, ACCESSION_HEADER) + metadataKeys + fileColumnHeaders
         }
 
         TsvWriter(outputStream, headers).use { writer ->
             for ((index, entry) in data.withIndex()) {
                 val metadataValues = metadataKeys.map { entry.submittedData.metadata[it] ?: "" }
+                val fileValues = fileCategories.map { category ->
+                    formatFilesCell(entry.submittedData.files?.get(category.name))
+                }
                 val row = if (isMultiSegmented) {
                     val fastaIds = fastaIdsByEntry[index].joinedUniqueFastaIds(FASTA_IDS_SEPARATOR)
-                    listOf(metadataIds[index], entry.accession, fastaIds) + metadataValues
+                    listOf(metadataIds[index], entry.accession, fastaIds) + metadataValues + fileValues
                 } else {
-                    listOf(metadataIds[index], entry.accession) + metadataValues
+                    listOf(metadataIds[index], entry.accession) + metadataValues + fileValues
                 }
                 writer.writeRow(row)
             }

backend/src/main/kotlin/org/loculus/backend/utils/MetadataEntry.kt

@@ -4,21 +4,27 @@ import org.apache.commons.csv.CSVException
 import org.apache.commons.csv.CSVFormat
 import org.apache.commons.csv.CSVParser
 import org.apache.commons.csv.CSVRecord
+import org.loculus.backend.api.FileCategoryFilesMap
+import org.loculus.backend.api.FileIdAndName
 import org.loculus.backend.controller.UnprocessableEntityException
 import org.loculus.backend.model.ACCESSION_HEADER
 import org.loculus.backend.model.FASTA_IDS_HEADER
 import org.loculus.backend.model.FASTA_IDS_SEPARATOR
+import org.loculus.backend.model.FILES_HEADER_PREFIX
+import org.loculus.backend.model.FILES_SEPARATOR
 import org.loculus.backend.model.FastaId
 import org.loculus.backend.model.METADATA_ID_HEADER
 import org.loculus.backend.model.METADATA_ID_HEADER_ALTERNATE_FOR_BACKCOMPAT
 import org.loculus.backend.model.SubmissionId
 import java.io.InputStream
 import java.io.InputStreamReader
+import java.util.UUID
 
 data class MetadataEntry(
     val submissionId: SubmissionId,
     val metadata: Map<String, String>,
     val fastaIds: Set<FastaId>? = null,
+    val files: FileCategoryFilesMap? = null,
 )
 
 private fun invalidTsvFormatException(originalException: Exception) = UnprocessableEntityException(
@@ -78,6 +84,78 @@ fun extractAndValidateFastaIds(record: CSVRecord, submissionId: String, recordNu
     }
 }
 
+/**
+ * Parses the `files.<category>` columns of a record into a [FileCategoryFilesMap].
+ * Each cell is a space-separated list of `fileName:fileId` pairs, e.g. `reads_1.fq:<uuid> reads_2.fq:<uuid>`.
+ * Returns `null` if the metadata file has no `files.*` columns at all. Categories with a blank cell are omitted.
+ */
+fun extractAndValidateFiles(record: CSVRecord, submissionId: String, recordNumber: Int): FileCategoryFilesMap? {
+    val fileHeaders = record.parser.headerNames.filter { it.startsWith(FILES_HEADER_PREFIX) }
+    if (fileHeaders.isEmpty()) {
+        return null
+    }
+
+    return fileHeaders.mapNotNull { header ->
+        val cellValue = record[header]
+        if (cellValue.isNullOrEmpty()) {
+            return@mapNotNull null
+        }
+
+        val category = header.removePrefix(FILES_HEADER_PREFIX)
+        val files = cellValue.split(FILES_SEPARATOR)
+            .map { it.trim() }
+            .filter { it.isNotEmpty() }
+            .map { token -> extractAndValidateFileIdAndName(token, header, submissionId, recordNumber) }
+
+        val duplicateNames = files.groupingBy { it.name }.eachCount().filter { it.value > 1 }.keys
+        if (duplicateNames.isNotEmpty()) {
+            throw UnprocessableEntityException(
+                "In metadata file: record #$recordNumber with id '$submissionId': " +
+                    "found duplicate file names in column '$header': " + duplicateNames.joinToString(", "),
+            )
+        }
+
+        category to files
+    }.toMap().ifEmpty { null }
+}
+
+private fun extractAndValidateFileIdAndName(
+    token: String,
+    header: String,
+    submissionId: String,
+    recordNumber: Int,
+): FileIdAndName {
+    // We currently still support ':' characters in file names, so take the last occurence.
+    // TODO: Update when file character list is restricted
+    val separatorIndex = token.lastIndexOf(':')
+    if (separatorIndex < 0) {
+        throw UnprocessableEntityException(
+            "In metadata file: record #$recordNumber with id '$submissionId': " +
+                "file entry '$token' in column '$header' is missing a file ID. Expected format 'fileName:fileId'.",
+        )
+    }
+    val name = token.substring(0, separatorIndex)
+    val fileIdString = token.substring(separatorIndex + 1)
+    if (name.isEmpty()) {
+        throw UnprocessableEntityException(
+            "In metadata file: record #$recordNumber with id '$submissionId': " +
+                "file entry '$token' in column '$header' is missing a file name. Expected format 'fileName:fileId'.",
+        )
+    }
+
+    // TODO: Update when moving away from UUIDs to more user-friendly file IDs
+    val fileId = try {
+        UUID.fromString(fileIdString)
+    } catch (e: IllegalArgumentException) {
+        throw UnprocessableEntityException(
+            "In metadata file: record #$recordNumber with id '$submissionId': " +
+                "file entry '$token' in column '$header' has an invalid file ID '$fileIdString'. " +
+                "Expected a UUID.",
+        )
+    }
+    return FileIdAndName(fileId, name)
+}
+
 private fun setUpCsvParser(metadataInputStream: InputStream): CSVParser {
     val csvParser = try {
         CSVFormat.TDF.builder().setHeader().setSkipHeaderRecord(true).get()
@@ -135,15 +213,17 @@ fun metadataEntryStreamAsSequence(metadataInputStream: InputStream): Sequence<Me
                 val submissionId = getValueAndValidateNoWhitespace(record, submissionIdHeader, recordNumber)
 
                 val fastaIds = extractAndValidateFastaIds(record, submissionId, recordNumber)
+                val files = extractAndValidateFiles(record, submissionId, recordNumber)
 
                 val metadata = record.toMap().filterKeys {
                     it != submissionIdHeader &&
-                        it != FASTA_IDS_HEADER
+                        it != FASTA_IDS_HEADER &&
+                        !it.startsWith(FILES_HEADER_PREFIX)
                 }
 
                 validateMetadataNotEmpty(metadata, submissionId, recordNumber)
 
-                yield(MetadataEntry(submissionId, metadata, fastaIds))
+                yield(MetadataEntry(submissionId, metadata, fastaIds, files))
             }
         } catch (e: java.io.UncheckedIOException) {
             throwWithCsvExceptionUnwrapped(e)
@@ -156,6 +236,7 @@ data class RevisionEntry(
     val accession: Accession,
     val metadata: Map<String, String>,
     val fastaIds: Set<FastaId>? = null,
+    val files: FileCategoryFilesMap? = null,
 )
 
 fun revisionEntryStreamAsSequence(metadataInputStream: InputStream): Sequence<RevisionEntry> {
@@ -179,14 +260,16 @@ fun revisionEntryStreamAsSequence(metadataInputStream: InputStream): Sequence<Re
                 val accession = getValueAndValidateNoWhitespace(record, ACCESSION_HEADER, recordNumber)
 
                 val fastaIds = extractAndValidateFastaIds(record, submissionId, recordNumber)
+                val files = extractAndValidateFiles(record, submissionId, recordNumber)
 
                 val metadata = record.toMap().filterKeys {
                     it != submissionIdHeader && it != ACCESSION_HEADER &&
-                        it != FASTA_IDS_HEADER
+                        it != FASTA_IDS_HEADER &&
+                        !it.startsWith(FILES_HEADER_PREFIX)
                 }
                 validateMetadataNotEmpty(metadata, submissionId, recordNumber)
 
-                yield(RevisionEntry(submissionId, accession, metadata, fastaIds))
+                yield(RevisionEntry(submissionId, accession, metadata, fastaIds, files))
             }
         } catch (e: java.io.UncheckedIOException) {
             throwWithCsvExceptionUnwrapped(e)