diff --git a/backend/src/main/kotlin/org/loculus/backend/api/SubmissionTypes.kt b/backend/src/main/kotlin/org/loculus/backend/api/SubmissionTypes.kt index 4ee737b238..38b536ce26 100644 --- a/backend/src/main/kotlin/org/loculus/backend/api/SubmissionTypes.kt +++ b/backend/src/main/kotlin/org/loculus/backend/api/SubmissionTypes.kt @@ -478,3 +478,8 @@ fun FileCategoryFilesMap.getDuplicateFileNames(category: FileCategory): Set 1 }.keys } + +fun FileCategoryFilesMap.mergeFileCategories(override: FileCategoryFilesMap?): FileCategoryFilesMap { + if (override == null) return this + return this + override +} diff --git a/backend/src/main/kotlin/org/loculus/backend/controller/SubmissionController.kt b/backend/src/main/kotlin/org/loculus/backend/controller/SubmissionController.kt index fc7a906fc7..8c22ebadcc 100644 --- a/backend/src/main/kotlin/org/loculus/backend/controller/SubmissionController.kt +++ b/backend/src/main/kotlin/org/loculus/backend/controller/SubmissionController.kt @@ -527,6 +527,11 @@ open class SubmissionController( val instanceConfig = backendConfig.getInstanceConfig(organism) val hasConsensusSequences = instanceConfig.schema.submissionDataTypes.consensusSequences val isMultiSegmented = instanceConfig.referenceGenome.nucleotideSequences.size > 1 + val fileCategories = if (instanceConfig.schema.submissionDataTypes.files.enabled) { + instanceConfig.schema.submissionDataTypes.files.categories + } else { + emptyList() + } val streamBody = StreamingResponseBody { responseBodyStream -> val startTime = System.currentTimeMillis() @@ -556,6 +561,7 @@ open class SubmissionController( uniqueFastaIdsByEntry, zipOut, isMultiSegmented, + fileCategories, ) zipOut.closeEntry() diff --git a/backend/src/main/kotlin/org/loculus/backend/model/SubmitModel.kt b/backend/src/main/kotlin/org/loculus/backend/model/SubmitModel.kt index 8fec3462e4..840456fe35 100644 --- a/backend/src/main/kotlin/org/loculus/backend/model/SubmitModel.kt +++ b/backend/src/main/kotlin/org/loculus/backend/model/SubmitModel.kt @@ -33,6 +33,9 @@ const val METADATA_ID_HEADER_ALTERNATE_FOR_BACKCOMPAT = "submissionId" const val FASTA_IDS_HEADER = "fastaIds" const val FASTA_IDS_SEPARATOR = " " +const val FILES_HEADER_PREFIX = "files." +const val FILES_SEPARATOR = " " + const val ACCESSION_HEADER = "accession" private val log = KotlinLogging.logger { } @@ -144,10 +147,20 @@ class SubmitModel( ) } - submissionParams.files?.let { submittedFiles -> - val fileSubmissionIds = submittedFiles.keys - validateSubmissionIdSetsForFiles(metadataSubmissionIds, fileSubmissionIds) - validateFileGroupOwnership(submittedFiles, submissionParams, uploadId) + // File mappings in submissionParams can contain submission Ids not present in the metadata + // This is implicitly validated for the file mappings within the metadata column + // TODO: This can be removed once file mappings JSON support is removed + submissionParams.files?.let { validateSubmissionIdSetsForFiles(metadataSubmissionIds, it.keys) } + + val files = uploadDatabaseService.getFilesForUpload(uploadId) + if (files.isNotEmpty()) { + submissionIdFilesMappingPreconditionValidator + .validateFilenameCharacters(files) + .validateFilenamesAreUnique(files) + .validateCategoriesMatchSchema(files, submissionParams.organism) + .validateMultipartUploads(files) + .validateFilesExist(files) + validateFileGroupOwnership(files, submissionParams, uploadId) } if (submissionParams is SubmissionParams.OriginalSubmissionParams) { diff --git a/backend/src/main/kotlin/org/loculus/backend/service/submission/UploadDatabaseService.kt b/backend/src/main/kotlin/org/loculus/backend/service/submission/UploadDatabaseService.kt index b4a913ed30..9c71a958d4 100644 --- a/backend/src/main/kotlin/org/loculus/backend/service/submission/UploadDatabaseService.kt +++ b/backend/src/main/kotlin/org/loculus/backend/service/submission/UploadDatabaseService.kt @@ -16,6 +16,7 @@ import org.loculus.backend.api.Organism import org.loculus.backend.api.Status import org.loculus.backend.api.SubmissionIdFilesMap import org.loculus.backend.api.SubmissionIdMapping +import org.loculus.backend.api.mergeFileCategories import org.loculus.backend.auth.AuthenticatedUser import org.loculus.backend.controller.UnprocessableEntityException import org.loculus.backend.log.AuditLogger @@ -81,7 +82,7 @@ class UploadDatabaseService( this[submissionIdColumn] = it.submissionId this[fastaIdsColumn] = it.fastaIds?.toList() this[metadataColumn] = it.metadata - this[filesColumn] = files?.get(it.submissionId) + this[filesColumn] = files?.get(it.submissionId)?.mergeFileCategories(it.files) ?: it.files this[organismColumn] = submittedOrganism.name this[uploadIdColumn] = uploadId } @@ -120,7 +121,7 @@ class UploadDatabaseService( this[submissionIdColumn] = it.submissionId this[fastaIdsColumn] = it.fastaIds?.toList() this[metadataColumn] = it.metadata - this[filesColumn] = files?.get(it.submissionId) + this[filesColumn] = files?.get(it.submissionId)?.mergeFileCategories(it.files) ?: it.files this[organismColumn] = submittedOrganism.name this[uploadIdColumn] = uploadId } @@ -166,6 +167,15 @@ class UploadDatabaseService( .where { uploadIdColumn eq uploadId } .map { it[submissionIdColumn] } + fun getFilesForUpload(uploadId: String): SubmissionIdFilesMap = MetadataUploadAuxTable + .select( + submissionIdColumn, + filesColumn, + ) + .where { uploadIdColumn eq uploadId } + .mapNotNull { row -> row[filesColumn]?.let { row[submissionIdColumn] to it } } + .toMap() + fun getFastaIdsForMetadata(uploadId: String): List> = MetadataUploadAuxTable .select( uploadIdColumn, diff --git a/backend/src/main/kotlin/org/loculus/backend/utils/GetSubmittedDataHelpers.kt b/backend/src/main/kotlin/org/loculus/backend/utils/GetSubmittedDataHelpers.kt index 7fd876d9d5..7a5486f73e 100644 --- a/backend/src/main/kotlin/org/loculus/backend/utils/GetSubmittedDataHelpers.kt +++ b/backend/src/main/kotlin/org/loculus/backend/utils/GetSubmittedDataHelpers.kt @@ -1,9 +1,13 @@ package org.loculus.backend.utils +import org.loculus.backend.api.FileIdAndName import org.loculus.backend.api.SubmittedDataDownloadEntry +import org.loculus.backend.config.FileCategory import org.loculus.backend.model.ACCESSION_HEADER import org.loculus.backend.model.FASTA_IDS_HEADER import org.loculus.backend.model.FASTA_IDS_SEPARATOR +import org.loculus.backend.model.FILES_HEADER_PREFIX +import org.loculus.backend.model.FILES_SEPARATOR import org.loculus.backend.model.FastaId import org.loculus.backend.model.METADATA_ID_HEADER import kotlin.collections.component1 @@ -20,6 +24,9 @@ data class UniqueFastaIdsForEntry(val uniqueFastaIdByOriginalFastaId: Map?): String = + files.orEmpty().joinToString(FILES_SEPARATOR) { "${it.name}:${it.fileId}" } + object GetSubmittedDataHelpers { fun uniqueFastaIdsByEntry( @@ -47,22 +54,27 @@ object GetSubmittedDataHelpers { fastaIdsByEntry: List, outputStream: java.io.OutputStream, isMultiSegmented: Boolean, + fileCategories: List, ) { val metadataKeys = data.flatMapTo(mutableSetOf()) { it.submittedData.metadata.keys }.sorted() + val fileColumnHeaders = fileCategories.map { "$FILES_HEADER_PREFIX${it.name}" } val headers = if (isMultiSegmented) { - listOf(METADATA_ID_HEADER, ACCESSION_HEADER, FASTA_IDS_HEADER) + metadataKeys + listOf(METADATA_ID_HEADER, ACCESSION_HEADER, FASTA_IDS_HEADER) + metadataKeys + fileColumnHeaders } else { - listOf(METADATA_ID_HEADER, ACCESSION_HEADER) + metadataKeys + listOf(METADATA_ID_HEADER, ACCESSION_HEADER) + metadataKeys + fileColumnHeaders } TsvWriter(outputStream, headers).use { writer -> for ((index, entry) in data.withIndex()) { val metadataValues = metadataKeys.map { entry.submittedData.metadata[it] ?: "" } + val fileValues = fileCategories.map { category -> + formatFilesCell(entry.submittedData.files?.get(category.name)) + } val row = if (isMultiSegmented) { val fastaIds = fastaIdsByEntry[index].joinedUniqueFastaIds(FASTA_IDS_SEPARATOR) - listOf(metadataIds[index], entry.accession, fastaIds) + metadataValues + listOf(metadataIds[index], entry.accession, fastaIds) + metadataValues + fileValues } else { - listOf(metadataIds[index], entry.accession) + metadataValues + listOf(metadataIds[index], entry.accession) + metadataValues + fileValues } writer.writeRow(row) } diff --git a/backend/src/main/kotlin/org/loculus/backend/utils/MetadataEntry.kt b/backend/src/main/kotlin/org/loculus/backend/utils/MetadataEntry.kt index 96cf5a5039..614957bc32 100644 --- a/backend/src/main/kotlin/org/loculus/backend/utils/MetadataEntry.kt +++ b/backend/src/main/kotlin/org/loculus/backend/utils/MetadataEntry.kt @@ -4,21 +4,27 @@ import org.apache.commons.csv.CSVException import org.apache.commons.csv.CSVFormat import org.apache.commons.csv.CSVParser import org.apache.commons.csv.CSVRecord +import org.loculus.backend.api.FileCategoryFilesMap +import org.loculus.backend.api.FileIdAndName import org.loculus.backend.controller.UnprocessableEntityException import org.loculus.backend.model.ACCESSION_HEADER import org.loculus.backend.model.FASTA_IDS_HEADER import org.loculus.backend.model.FASTA_IDS_SEPARATOR +import org.loculus.backend.model.FILES_HEADER_PREFIX +import org.loculus.backend.model.FILES_SEPARATOR import org.loculus.backend.model.FastaId import org.loculus.backend.model.METADATA_ID_HEADER import org.loculus.backend.model.METADATA_ID_HEADER_ALTERNATE_FOR_BACKCOMPAT import org.loculus.backend.model.SubmissionId import java.io.InputStream import java.io.InputStreamReader +import java.util.UUID data class MetadataEntry( val submissionId: SubmissionId, val metadata: Map, val fastaIds: Set? = null, + val files: FileCategoryFilesMap? = null, ) private fun invalidTsvFormatException(originalException: Exception) = UnprocessableEntityException( @@ -78,6 +84,78 @@ fun extractAndValidateFastaIds(record: CSVRecord, submissionId: String, recordNu } } +/** + * Parses the `files.` columns of a record into a [FileCategoryFilesMap]. + * Each cell is a space-separated list of `fileName:fileId` pairs, e.g. `reads_1.fq: reads_2.fq:`. + * Returns `null` if the metadata file has no `files.*` columns at all. Categories with a blank cell are omitted. + */ +fun extractAndValidateFiles(record: CSVRecord, submissionId: String, recordNumber: Int): FileCategoryFilesMap? { + val fileHeaders = record.parser.headerNames.filter { it.startsWith(FILES_HEADER_PREFIX) } + if (fileHeaders.isEmpty()) { + return null + } + + return fileHeaders.mapNotNull { header -> + val cellValue = record[header] + if (cellValue.isNullOrEmpty()) { + return@mapNotNull null + } + + val category = header.removePrefix(FILES_HEADER_PREFIX) + val files = cellValue.split(FILES_SEPARATOR) + .map { it.trim() } + .filter { it.isNotEmpty() } + .map { token -> extractAndValidateFileIdAndName(token, header, submissionId, recordNumber) } + + val duplicateNames = files.groupingBy { it.name }.eachCount().filter { it.value > 1 }.keys + if (duplicateNames.isNotEmpty()) { + throw UnprocessableEntityException( + "In metadata file: record #$recordNumber with id '$submissionId': " + + "found duplicate file names in column '$header': " + duplicateNames.joinToString(", "), + ) + } + + category to files + }.toMap().ifEmpty { null } +} + +private fun extractAndValidateFileIdAndName( + token: String, + header: String, + submissionId: String, + recordNumber: Int, +): FileIdAndName { + // We currently still support ':' characters in file names, so take the last occurence. + // TODO: Update when file character list is restricted + val separatorIndex = token.lastIndexOf(':') + if (separatorIndex < 0) { + throw UnprocessableEntityException( + "In metadata file: record #$recordNumber with id '$submissionId': " + + "file entry '$token' in column '$header' is missing a file ID. Expected format 'fileName:fileId'.", + ) + } + val name = token.substring(0, separatorIndex) + val fileIdString = token.substring(separatorIndex + 1) + if (name.isEmpty()) { + throw UnprocessableEntityException( + "In metadata file: record #$recordNumber with id '$submissionId': " + + "file entry '$token' in column '$header' is missing a file name. Expected format 'fileName:fileId'.", + ) + } + + // TODO: Update when moving away from UUIDs to more user-friendly file IDs + val fileId = try { + UUID.fromString(fileIdString) + } catch (e: IllegalArgumentException) { + throw UnprocessableEntityException( + "In metadata file: record #$recordNumber with id '$submissionId': " + + "file entry '$token' in column '$header' has an invalid file ID '$fileIdString'. " + + "Expected a UUID.", + ) + } + return FileIdAndName(fileId, name) +} + private fun setUpCsvParser(metadataInputStream: InputStream): CSVParser { val csvParser = try { CSVFormat.TDF.builder().setHeader().setSkipHeaderRecord(true).get() @@ -135,15 +213,17 @@ fun metadataEntryStreamAsSequence(metadataInputStream: InputStream): Sequence, val fastaIds: Set? = null, + val files: FileCategoryFilesMap? = null, ) fun revisionEntryStreamAsSequence(metadataInputStream: InputStream): Sequence { @@ -179,14 +260,16 @@ fun revisionEntryStreamAsSequence(metadataInputStream: InputStream): Sequence { + metadataEntryStreamAsSequence(ByteArrayInputStream(str.toByteArray())).toList() + } + assertThat(exception.message, containsString("missing a file ID")) + } + + @Test + fun `test files entry missing file name is rejected`() { + val fileId1 = "123e4567-e89b-12d3-a456-426614174000" + val str = """ + submissionId${'\t'}files.raw_reads${'\t'}Country + foo${'\t'}:$fileId1${'\t'}bar + """.trimIndent() + val exception = assertThrows { + metadataEntryStreamAsSequence(ByteArrayInputStream(str.toByteArray())).toList() + } + assertThat(exception.message, containsString("missing a file name")) + } + + @Test + fun `test files entry with invalid UUID is rejected`() { + val str = """ + submissionId${'\t'}files.raw_reads${'\t'}Country + foo${'\t'}reads_1.fq:not-a-uuid${'\t'}bar + """.trimIndent() + val exception = assertThrows { + metadataEntryStreamAsSequence(ByteArrayInputStream(str.toByteArray())).toList() + } + assertThat(exception.message, containsString("invalid file ID")) + } + + @Test + fun `test duplicate file names within a category are rejected`() { + val fileId1 = "123e4567-e89b-12d3-a456-426614174000" + val fileId2 = "223e4567-e89b-12d3-a456-426614174001" + val str = """ + submissionId${'\t'}files.raw_reads${'\t'}Country + foo${'\t'}reads.fq:$fileId1 reads.fq:$fileId2${'\t'}bar + """.trimIndent() + val exception = assertThrows { + metadataEntryStreamAsSequence(ByteArrayInputStream(str.toByteArray())).toList() + } + assertThat(exception.message, containsString("duplicate file names")) + assertThat(exception.message, containsString("reads.fq")) + } + + @Test + fun `test file name containing a colon splits on the last colon`() { + val fileId1 = "123e4567-e89b-12d3-a456-426614174000" + val str = """ + submissionId${'\t'}files.raw_reads${'\t'}Country + foo${'\t'}weird:name.fq:$fileId1${'\t'}bar + """.trimIndent() + val entries = metadataEntryStreamAsSequence(ByteArrayInputStream(str.toByteArray())).toList() + assertThat( + entries[0].files!!["raw_reads"], + equalTo(listOf(FileIdAndName(UUID.fromString(fileId1), "weird:name.fq"))), + ) + } } class RevisionEntryTest { @@ -304,4 +440,19 @@ class RevisionEntryTest { assertThat(exception.message, containsString("duplicate fasta ids")) assertThat(exception.message, containsString("seq1")) } + + @Test + fun `test revision files column is parsed and excluded from metadata`() { + val fileId1 = "123e4567-e89b-12d3-a456-426614174000" + val str = """ + submissionId${'\t'}accession${'\t'}files.raw_reads${'\t'}Country + foo${'\t'}ACC123${'\t'}reads.fq:$fileId1${'\t'}bar + """.trimIndent() + val entries = revisionEntryStreamAsSequence(ByteArrayInputStream(str.toByteArray())).toList() + assertThat( + entries[0].files, + equalTo(mapOf("raw_reads" to listOf(FileIdAndName(UUID.fromString(fileId1), "reads.fq")))), + ) + assertThat(entries[0].metadata.containsKey("files.raw_reads"), equalTo(false)) + } } diff --git a/integration-tests/tests/specs/features/file-sharing.spec.ts b/integration-tests/tests/specs/features/file-sharing.spec.ts index 3da863d164..d798c47b99 100644 --- a/integration-tests/tests/specs/features/file-sharing.spec.ts +++ b/integration-tests/tests/specs/features/file-sharing.spec.ts @@ -9,6 +9,7 @@ import { BulkSubmissionPage, SingleSequenceSubmissionPage } from '../../pages/su const ORGANISM_NAME = 'Test organism (with files)'; const ORGANISM_URL_NAME = 'dummy-organism-with-files'; const RAW_READS = 'raw_reads'; +const RAW_READS_FILES_HEADER = `files.${RAW_READS}`; const METADATA_HEADERS = ['submissionId', 'country', 'date']; const COUNTRY_1 = 'Norway'; const COUNTRY_2 = 'Uganda'; @@ -17,6 +18,12 @@ const ID_2 = 'sub2'; const FILES_SINGLE = { 'testfile.txt': 'This is a test file.' }; const FILES_DOUBLE = { 'file1.txt': 'Content of file 1.', 'file2.txt': 'Content of file 2.' }; +// Tests upload files in subfolders grouped by submissionId +const filesColumnCell = (submissionId: string, files: Record) => + Object.keys(files) + .map((name) => `${name}::${submissionId}/${name}`) + .join(' '); + test('submit single seq w/ 2 files thru single seq submission form', async ({ page, groupId, @@ -44,10 +51,13 @@ test('bulk submit 2 seqs with 1 & 2 files respectively', async ({ page, groupId, void groupId; const submissionPage = new BulkSubmissionPage(page); await submissionPage.navigateToSubmissionPage(ORGANISM_NAME); - await submissionPage.uploadMetadataFile(METADATA_HEADERS, [ - [ID_1, COUNTRY_1, '2022-12-02'], - [ID_2, COUNTRY_2, '2022-12-13'], - ]); + await submissionPage.uploadMetadataFile( + [...METADATA_HEADERS, RAW_READS_FILES_HEADER], + [ + [ID_1, COUNTRY_1, '2022-12-02', filesColumnCell(ID_1, FILES_SINGLE)], + [ID_2, COUNTRY_2, '2022-12-13', filesColumnCell(ID_2, FILES_DOUBLE)], + ], + ); await submissionPage.uploadExternalFiles( RAW_READS, { [ID_1]: FILES_SINGLE, [ID_2]: FILES_DOUBLE }, @@ -64,7 +74,10 @@ test('bulk submit 1 seq: discarding and readding a file', async ({ page, groupId void groupId; const submissionPage = new BulkSubmissionPage(page); await submissionPage.navigateToSubmissionPage(ORGANISM_NAME); - await submissionPage.uploadMetadataFile(METADATA_HEADERS, [[ID_1, COUNTRY_1, '2023-01-01']]); + await submissionPage.uploadMetadataFile( + [...METADATA_HEADERS, RAW_READS_FILES_HEADER], + [[ID_1, COUNTRY_1, '2023-01-01', filesColumnCell(ID_1, FILES_DOUBLE)]], + ); await submissionPage.uploadExternalFiles(RAW_READS, { [ID_1]: FILES_SINGLE }, tmpDir); await submissionPage.discardRawReadsFiles(); await submissionPage.uploadExternalFiles(RAW_READS, { [ID_1]: FILES_DOUBLE }, tmpDir); @@ -88,7 +101,10 @@ test('bulk submit 1 seq with a 35 MB file', async ({ page, groupId, tmpDir }) => const submissionPage = new BulkSubmissionPage(page); await submissionPage.navigateToSubmissionPage(ORGANISM_NAME); - await submissionPage.uploadMetadataFile(METADATA_HEADERS, [[ID_1, COUNTRY_1, '2024-01-01']]); + await submissionPage.uploadMetadataFile( + [...METADATA_HEADERS, RAW_READS_FILES_HEADER], + [[ID_1, COUNTRY_1, '2024-01-01', filesColumnCell(ID_1, LARGE_FILE)]], + ); await submissionPage.uploadExternalFiles(RAW_READS, { [ID_1]: LARGE_FILE }, tmpDir); const reviewPage = await submissionPage.submitAndWaitForProcessingDone(); const searchPage = await reviewPage.releaseAndGoToReleasedSequences(); diff --git a/website/src/components/Edit/EditPage.tsx b/website/src/components/Edit/EditPage.tsx index fe3768cb5a..36a1972442 100644 --- a/website/src/components/Edit/EditPage.tsx +++ b/website/src/components/Edit/EditPage.tsx @@ -9,7 +9,7 @@ import { getClientLogger } from '../../clientLogger.ts'; import { routes } from '../../routes/routes.ts'; import { backendApi } from '../../services/backendApi.ts'; import { backendClientHooks } from '../../services/serviceHooks.ts'; -import { type FilesBySubmissionId, type SequenceEntryToEdit, approvedForReleaseStatus } from '../../types/backend.ts'; +import { type SequenceEntryToEdit, approvedForReleaseStatus } from '../../types/backend.ts'; import { type InputField, type SubmissionDataTypes } from '../../types/config.ts'; import { getLatestAccessionVersionForRevision, type SequenceEntryHistory } from '../../types/lapis.ts'; import type { ClientConfig } from '../../types/runtimeConfig.ts'; @@ -18,6 +18,7 @@ import { getAccessionVersionString, parseAccessionVersionFromString } from '../. import { displayConfirmationDialog } from '../ConfirmationDialog.tsx'; import { SequenceEntryHistoryMenu } from '../SequenceDetailsPage/SequenceEntryHistoryMenu.tsx'; import { ExtraFilesUpload } from '../Submission/DataUploadForm.tsx'; +import { applyFileMappings, type FileMapping } from '../Submission/FileUpload/fileMapping.ts'; import { Button } from '../common/Button'; import ErrorBox from '../common/ErrorBox'; import { Spinner } from '../common/Spinner'; @@ -63,11 +64,16 @@ const InnerEditPage: FC = ({ ); const extraFilesEnabled = submissionDataTypes.files?.enabled ?? false; - const [fileMapping, setFileMapping] = useState(() => - extraFilesEnabled && dataToEdit.submittedData.files - ? { [dataToEdit.submissionId]: dataToEdit.submittedData.files } - : undefined, - ); + const [fileMapping, setFileMapping] = useState(() => { + const previousFiles = dataToEdit.submittedData.files; + if (!previousFiles) return undefined; + return new Map( + Object.entries(previousFiles).map(([category, files]) => [ + category, + new Map(files.map((file) => [file.name, { name: file.name, path: file.name, fileId: file.fileId }])), + ]), + ); + }); const isCreatingRevision = dataToEdit.status === approvedForReleaseStatus; @@ -87,9 +93,7 @@ const InnerEditPage: FC = ({ (message) => toast.error(message, { position: 'top-center', autoClose: false }), ); - const submitEditedDataForAccessionVersion = () => { - const fileMappingForSubmission = extraFilesEnabled ? fileMapping : undefined; - + const submitEditedDataForAccessionVersion = async () => { if (isCreatingRevision) { const fastaIds = submissionDataTypes.consensusSequences ? editableSequences.getFastaIds() : undefined; const metadataFile = editableMetadata.getMetadataTsv( @@ -102,10 +106,16 @@ const InnerEditPage: FC = ({ return; } + let finalMetadataFile = metadataFile; + const finalSubmissionFileMapping = new Map([[dataToEdit.submissionId, fileMapping ?? new Map()]]); + + if (extraFilesEnabled) { + finalMetadataFile = await applyFileMappings(metadataFile, finalSubmissionFileMapping); + } + if (!submissionDataTypes.consensusSequences) { submitRevision({ - metadataFile, - fileMapping: fileMappingForSubmission, + metadataFile: finalMetadataFile, }); return; } @@ -118,12 +128,19 @@ const InnerEditPage: FC = ({ return; } submitRevision({ - metadataFile, + metadataFile: finalMetadataFile, sequenceFile, - fileMapping: fileMappingForSubmission, }); } else { - const fileMappingForEdit = fileMappingForSubmission?.[dataToEdit.submissionId] ?? null; + const fileMappingForEdit = + extraFilesEnabled && fileMapping !== undefined + ? Object.fromEntries( + [...fileMapping].map(([category, files]) => [ + category, + [...files.values()].map((file) => ({ fileId: file.fileId!, name: file.name })), + ]), + ) + : null; submitEdit({ accession: dataToEdit.accession, version: dataToEdit.version, diff --git a/website/src/components/Submission/DataUploadForm.tsx b/website/src/components/Submission/DataUploadForm.tsx index 5f7d71fa32..77dbd38e39 100644 --- a/website/src/components/Submission/DataUploadForm.tsx +++ b/website/src/components/Submission/DataUploadForm.tsx @@ -15,7 +15,7 @@ import { type Group, openDataUseTermsOption, restrictedDataUseTermsOption, - type FilesBySubmissionId, + // type FilesBySubmissionId, } from '../../types/backend.ts'; import type { FileCategory, InputField } from '../../types/config.ts'; import type { SubmissionDataTypes } from '../../types/config.ts'; @@ -28,6 +28,12 @@ import { Button } from '../common/Button'; import { Checkbox } from '../common/Checkbox'; import { Spinner } from '../common/Spinner'; import { withQueryProvider } from '../common/withQueryProvider.tsx'; +import { + applyFileMappings, + mergeFileMappings, + type FileMapping, + type SubmissionFileMapping, +} from './FileUpload/fileMapping.ts'; export type UploadAction = 'submit' | 'revise'; @@ -66,7 +72,8 @@ const InnerDataUploadForm = ({ const { submit, revise, isPending } = useSubmitFiles(accessToken, organism, clientConfig, onSuccess, onError); const [fileFactory, setFileFactory] = useState(undefined); - const [fileMapping, setFileMapping] = useState(undefined); + const [fileMapping, setFileMapping] = useState(undefined); + const [submissionFileMapping, setSubmissionFileMapping] = useState(undefined); const [dataUseTermsType, setDataUseTermsType] = useState(openDataUseTermsOption); const [restrictedUntil, setRestrictedUntil] = useState(dateTimeInMonths(6)); @@ -103,10 +110,20 @@ const InnerDataUploadForm = ({ return; } - let fileMappingWithSubmissionId = fileMapping; - // for single submission, use the submissionID that the user gave in the form - if (extraFilesEnabled && inputMode === 'form' && fileMapping !== undefined) { - fileMappingWithSubmissionId = { [submissionId!]: Object.values(fileMapping)[0] }; + let finalMetadataFile = metadataFile; + let finalSubmissionFileMapping = submissionFileMapping; + + if (submissionId !== undefined && inputMode === 'form') { + finalSubmissionFileMapping = new Map([[submissionId, fileMapping ?? new Map()]]); + } + + if (extraFilesEnabled && finalSubmissionFileMapping !== undefined) { + const merged = mergeFileMappings(finalSubmissionFileMapping, fileMapping ?? new Map()); + if (merged.isErr()) { + onError(merged.error.message); + return; + } + finalMetadataFile = await applyFileMappings(metadataFile, merged.value); } const submitSequenceData = () => { @@ -114,9 +131,8 @@ const InnerDataUploadForm = ({ case 'submit': { const groupId = group.groupId; submit({ - metadataFile: metadataFile, + metadataFile: finalMetadataFile, sequenceFile: sequenceFile, - fileMapping: extraFilesEnabled ? fileMappingWithSubmissionId : undefined, groupId, dataUseTermsType, restrictedUntil: @@ -128,9 +144,8 @@ const InnerDataUploadForm = ({ } case 'revise': revise({ - metadataFile: metadataFile, + metadataFile: finalMetadataFile, sequenceFile: sequenceFile, - fileMapping: extraFilesEnabled ? fileMappingWithSubmissionId : undefined, }); break; } @@ -162,10 +177,12 @@ const InnerDataUploadForm = ({
{extraFilesEnabled && ( @@ -179,6 +196,7 @@ const InnerDataUploadForm = ({ onError={onError} fileMapping={fileMapping} setFileMapping={setFileMapping} + submissionFileMapping={submissionFileMapping} />
@@ -281,6 +299,7 @@ export const ExtraFilesUpload = ({ fileCategories, fileMapping, setFileMapping, + submissionFileMapping, formSubmissionId, onError, }: { @@ -289,8 +308,9 @@ export const ExtraFilesUpload = ({ inputMode: InputMode; groupId: number; fileCategories: FileCategory[]; - fileMapping: FilesBySubmissionId | undefined; - setFileMapping: Dispatch>; + fileMapping: FileMapping | undefined; + setFileMapping: Dispatch>; + submissionFileMapping?: SubmissionFileMapping | undefined; formSubmissionId?: string; onError: (message: string) => void; }) => { @@ -316,6 +336,7 @@ export const ExtraFilesUpload = ({ onError={onError} fileMapping={fileMapping} setFileMapping={setFileMapping} + submissionFileMapping={submissionFileMapping} formSubmissionId={formSubmissionId} /> ))} diff --git a/website/src/components/Submission/FileUpload/ColumnMapping.ts b/website/src/components/Submission/FileUpload/ColumnMapping.ts index 0722faa897..f67667b136 100644 --- a/website/src/components/Submission/FileUpload/ColumnMapping.ts +++ b/website/src/components/Submission/FileUpload/ColumnMapping.ts @@ -1,5 +1,6 @@ import Papa from 'papaparse'; +import { FILES_HEADER_PREFIX } from './fileMapping'; import { type ProcessedFile } from './fileProcessing'; import type { InputField } from '../../../types/config'; import stringSimilarity from '../../../utils/stringSimilarity'; @@ -97,11 +98,21 @@ export class ColumnMapping { const headersInFile = inputRows.splice(0, 1)[0]; const headers: string[] = []; const indices: number[] = []; + const mappedSourceColumns = new Set(); this.entries().forEach(([sourceCol, targetCol]) => { if (targetCol === null) return; + mappedSourceColumns.add(sourceCol); headers.push(targetCol); indices.push(headersInFile.findIndex((sourceHeader) => sourceHeader === sourceCol)); }); + + // Include file category columns, as these will not be present in the column mapping + headersInFile.forEach((sourceHeader, sourceIndex) => { + if (sourceHeader.startsWith(FILES_HEADER_PREFIX) && !mappedSourceColumns.has(sourceHeader)) { + headers.push(sourceHeader); + indices.push(sourceIndex); + } + }); const newRows = inputRows.map((row) => indices.map((i) => row[i])); const newFileContent = Papa.unparse([headers, ...newRows], { delimiter: '\t', newline: '\n' }); return new File([newFileContent], 'remapped.tsv'); diff --git a/website/src/components/Submission/FileUpload/FolderUploadComponent.spec.tsx b/website/src/components/Submission/FileUpload/FolderUploadComponent.spec.tsx index b511fe5247..0d9acfc165 100644 --- a/website/src/components/Submission/FileUpload/FolderUploadComponent.spec.tsx +++ b/website/src/components/Submission/FileUpload/FolderUploadComponent.spec.tsx @@ -44,23 +44,24 @@ const defaultProps = { groupId: 1, fileMapping: undefined, setFileMapping: mockSetFileMapping, + submissionFileMapping: undefined, onError: mockOnError, }; -// In the case of previous uploads, they are keyed by a real submission id, not the dummySubmissionId +// The fileMapping is keyed by category, then by file path. Previous uploads have no upload path, +// so (within a single entry, where names are unique) they are keyed by and carry a path of their name. +const fileMappingOf = (files: { fileId: string; name: string }[]) => + new Map([['extraFiles', new Map(files.map((f) => [f.name, { name: f.name, path: f.name, fileId: f.fileId }]))]]); + const submissionId = 'SUBMISSION_ID_123'; const defaultPropsWithFiles = { ...defaultProps, inputMode: 'form' as const, formSubmissionId: submissionId, - fileMapping: { - [submissionId]: { - extraFiles: [ - { fileId: 'file-1', name: 'file-a.txt' }, - { fileId: 'file-2', name: 'file-b.txt' }, - ], - }, - }, + fileMapping: fileMappingOf([ + { fileId: 'file-1', name: 'file-a.txt' }, + { fileId: 'file-2', name: 'file-b.txt' }, + ]), }; describe('FolderUploadComponent', () => { @@ -209,11 +210,7 @@ describe('FolderUploadComponent', () => { it('reverts to the upload folder prompt after discarding the last upload individually', async () => { const singleFileProps = { ...defaultPropsWithFiles, - fileMapping: { - [submissionId]: { - extraFiles: [{ fileId: 'file-1', name: 'file-a.txt' }], - }, - }, + fileMapping: fileMappingOf([{ fileId: 'file-1', name: 'file-a.txt' }]), }; render(); @@ -241,7 +238,7 @@ describe('FolderUploadComponent', () => { , ); @@ -272,19 +269,6 @@ describe('FolderUploadComponent', () => { expect(screen.getByText('file-b.txt')).toBeInTheDocument(); }); - it('rejects additional files containing duplicate file names', async () => { - render(); - - const dup1 = new File(['a'], 'dup.txt', { type: 'text/plain' }); - const dup2 = new File(['b'], 'dup.txt', { type: 'text/plain' }); - Object.defineProperty(dup1, 'webkitRelativePath', { value: '', writable: false }); - Object.defineProperty(dup2, 'webkitRelativePath', { value: '', writable: false }); - await userEvent.upload(screen.getByTestId('add_extraFiles'), [dup1, dup2]); - - expect(mockOnError).toHaveBeenCalledWith(expect.stringContaining('dup.txt')); - expect(mockRequestMultipartUpload).not.toHaveBeenCalled(); - }); - it('confirms before overwriting an existing file with the same name', async () => { mockRequestMultipartUpload.mockReturnValue(ok([{ fileId: 'replacement-id', urls: ['http://test.com/url1'] }])); @@ -302,19 +286,6 @@ describe('FolderUploadComponent', () => { await waitFor(() => expect(mockRequestMultipartUpload).toHaveBeenCalled()); }); - it('does not show the additional files button in bulk mode', () => { - render( - , - ); - - expect(screen.getByText('file-a.txt')).toBeInTheDocument(); - expect(screen.queryByTestId('add_button_extraFiles')).not.toBeInTheDocument(); - }); - it('disables the additional files button while an upload is in progress', async () => { mockRequestMultipartUpload.mockReturnValue(ok([{ fileId: 'added-id', urls: ['http://test.com/url1'] }])); // Keep the upload pending so the component stays in the uploadInProgress state. diff --git a/website/src/components/Submission/FileUpload/FolderUploadComponent.tsx b/website/src/components/Submission/FileUpload/FolderUploadComponent.tsx index 46210275c8..f19686bdeb 100644 --- a/website/src/components/Submission/FileUpload/FolderUploadComponent.tsx +++ b/website/src/components/Submission/FileUpload/FolderUploadComponent.tsx @@ -1,10 +1,10 @@ import { produce } from 'immer'; -import { useEffect, useState, type Dispatch, type FC, type SetStateAction } from 'react'; +import { useEffect, useMemo, useState, type Dispatch, type FC, type SetStateAction } from 'react'; import { toast } from 'react-toastify'; +import type { FileMapping, SubmissionFileMapping } from './fileMapping'; import useClientFlag from '../../../hooks/isClient'; import { BackendClient } from '../../../services/backendClient'; -import type { FilesBySubmissionId } from '../../../types/backend'; import { type FileCategory } from '../../../types/config'; import type { ClientConfig } from '../../../types/runtimeConfig'; import { calculatePartSizeAndCount, splitFileIntoParts, uploadPart } from '../../../utils/multipartUpload'; @@ -15,47 +15,43 @@ import LucideFile from '~icons/lucide/file'; import LucideFolderUp from '~icons/lucide/folder-up'; import LucideLoader from '~icons/lucide/loader'; -type SubmissionId = string; - -const DUMMY_SUBMISSION_ID = 'dummySubmissionId'; - -type FileAndName = { - file: File; - name: string; -}; - /** * The state that the component is in, right after the user dropped the files. * We're awaiting the presigned upload URLs from the backend, to start uploading. */ type AwaitingUrlState = { type: 'awaitingUrls'; - files: Record; + files: Awaiting[]; }; -type SingleFileUpload = Pending | Uploaded | PreviousUpload | Error; - type UploadInProgressState = { type: 'uploadInProgress'; - files: Record; + files: SingleFileUpload[]; }; type UploadCompleted = { type: 'uploadCompleted'; - files: Record; + files: (Uploaded | PreviousUpload)[]; }; type FileUploadState = AwaitingUrlState | UploadInProgressState | UploadCompleted; type UploadStatus = 'pending' | 'uploaded' | 'previousUpload' | 'error'; +type Awaiting = { + type: 'awaiting'; + file: File; + name: string; + path: string; +}; + type Pending = { type: 'pending'; file: File; name: string; + path: string; size: number; fileId: string; - urls: string[]; uploadedParts: number; totalParts: number; @@ -67,6 +63,7 @@ type Uploaded = { type: 'uploaded'; fileId: string; name: string; + path: string; size: number; }; @@ -74,23 +71,28 @@ type PreviousUpload = { type: 'previousUpload'; fileId: string; name: string; + path: string; }; type Error = { type: 'error'; name: string; + path: string; size: number; msg: string; }; +type SingleFileUpload = Pending | Uploaded | PreviousUpload | Error; + type FolderUploadComponentProps = { fileCategory: FileCategory; inputMode: InputMode; accessToken: string; clientConfig: ClientConfig; groupId: number; - fileMapping: FilesBySubmissionId | undefined; - setFileMapping: Dispatch>; + fileMapping: FileMapping | undefined; + setFileMapping: Dispatch>; + submissionFileMapping: SubmissionFileMapping | undefined; // Passed when the submissionId is known (e.g. editing/revising an entry) in form mode, // where it is used instead of the dummySubmissionId placeholder. formSubmissionId?: string; @@ -105,37 +107,43 @@ export const FolderUploadComponent: FC = ({ groupId, fileMapping, setFileMapping, - formSubmissionId, + submissionFileMapping, onError, }) => { const isClient = useClientFlag(); const [fileUploadState, setFileUploadState] = useState(() => { - if (fileMapping === undefined) return undefined; - - const previousUploadFiles: Record = {}; - Object.entries(fileMapping).forEach(([submissionId, categories]) => { - const fileCategoryFiles = categories[fileCategory.name] ?? []; - previousUploadFiles[submissionId] = fileCategoryFiles.map((file) => ({ - type: 'previousUpload', - fileId: file.fileId, - name: file.name, - })); - }); - - const hasPreviousFiles = Object.values(previousUploadFiles).some((files) => files.length > 0); - if (!hasPreviousFiles) return undefined; - - return { type: 'uploadCompleted', files: previousUploadFiles }; + const categoryFiles = fileMapping?.get(fileCategory.name); + if (categoryFiles === undefined || categoryFiles.size === 0) return undefined; + + const files: PreviousUpload[] = [...categoryFiles.values()].map((file) => ({ + type: 'previousUpload', + fileId: file.fileId!, + name: file.name, + path: file.path, + })); + return { type: 'uploadCompleted', files }; }); const [isDragging, setIsDragging] = useState(false); const backendClient = new BackendClient(clientConfig.backendUrl); - function updatePartProgress(submissionId: string, fileId: string, uploadedParts: number, totalParts: number) { + const submissionFilePaths = useMemo(() => { + return submissionFileMapping + ? new Set( + Array.from(submissionFileMapping.values()).flatMap((categories) => + Array.from(categories.get(fileCategory.name)?.values() ?? []).map( + (submissionFile) => submissionFile.path, + ), + ), + ) + : undefined; + }, [fileCategory.name, submissionFileMapping]); + + function updatePartProgress(fileId: string, uploadedParts: number, totalParts: number) { setFileUploadState((state) => { if (state?.type === 'uploadInProgress') { return produce(state, (draft) => { - const file = draft.files[submissionId].find((f) => f.type === 'pending' && f.fileId === fileId); + const file = draft.files.find((f) => f.type === 'pending' && f.fileId === fileId); if (file?.type === 'pending') { file.uploadedParts = uploadedParts; file.totalParts = totalParts; @@ -146,16 +154,22 @@ export const FolderUploadComponent: FC = ({ }); } - function updateFileState(submissionId: string, fileId: string, newStatus: 'uploaded' | 'error', errorMsg?: string) { + function updateFileState(fileId: string, newStatus: 'uploaded' | 'error', errorMsg?: string) { setFileUploadState((state) => { if (state?.type === 'uploadInProgress') { return produce(state, (draft) => { - draft.files[submissionId] = state.files[submissionId].map((file) => { + draft.files = draft.files.map((file) => { if (file.type === 'pending' && file.fileId === fileId) { if (newStatus === 'uploaded') { - return { type: 'uploaded', fileId, name: file.name, size: file.size }; + return { type: 'uploaded', fileId, name: file.name, path: file.path, size: file.size }; } else { - return { type: 'error', name: file.name, size: file.size, msg: errorMsg! }; + return { + type: 'error', + name: file.name, + path: file.path, + size: file.size, + msg: errorMsg!, + }; } } return file; @@ -166,36 +180,34 @@ export const FolderUploadComponent: FC = ({ }); } - async function uploadMultipartFile(submissionId: string, pending: Pending) { + async function uploadMultipartFile(pending: Pending) { const parts = splitFileIntoParts(pending.file, pending.partSize); const etags: string[] = []; for (let i = 0; i < parts.length; i++) { const etag = await uploadPart(pending.urls[i], parts[i]); etags.push(etag); - updatePartProgress(submissionId, pending.fileId, i + 1, pending.totalParts); + updatePartProgress(pending.fileId, i + 1, pending.totalParts); } const result = await backendClient.completeMultipartUpload(accessToken, [{ fileId: pending.fileId, etags }]); result.match( - () => updateFileState(submissionId, pending.fileId, 'uploaded'), + () => updateFileState(pending.fileId, 'uploaded'), (err) => { - updateFileState(submissionId, pending.fileId, 'error', err.detail); + updateFileState(pending.fileId, 'error', err.detail); onError(err.detail); throw new Error(`Upload of file ${pending.fileId} failed: ${err.detail}`); }, ); } - async function startUploading(submissionIdFileMap: Record) { - for (const [submissionId, files] of Object.entries(submissionIdFileMap)) { - for (const pending of files) { - await uploadMultipartFile(submissionId, pending); - } + async function startUploading(pendingFiles: Pending[]) { + for (const pending of pendingFiles) { + await uploadMultipartFile(pending); } } - async function requestFileUploads(filesAwaitingUrls: FileAndName[]): Promise { + async function requestFileUploads(filesAwaitingUrls: Awaiting[]): Promise { const pendingFiles: Pending[] = []; for (const file of filesAwaitingUrls) { const { partCount, partSize } = calculatePartSizeAndCount(file.file.size); @@ -206,6 +218,7 @@ export const FolderUploadComponent: FC = ({ type: 'pending', file: file.file, name: file.name, + path: file.path, size: file.file.size, fileId: data[0].fileId, urls: data[0].urls, @@ -224,22 +237,10 @@ export const FolderUploadComponent: FC = ({ useEffect(() => { if (fileUploadState === undefined) { setFileMapping((currentMapping) => { - if (inputMode === 'bulk') { - if (currentMapping !== undefined) { - return produce(currentMapping, (draft) => { - Object.keys(draft).forEach((submissionId) => { - draft[submissionId][fileCategory.name] = []; - }); - }); - } else { - return undefined; - } - } else { - return produce(currentMapping ?? {}, (draft) => { - const submissionId = formSubmissionId ?? DUMMY_SUBMISSION_ID; - draft[submissionId] = { ...draft[submissionId], [fileCategory.name]: [] }; - }); - } + if (currentMapping === undefined) return undefined; + const newMapping = new Map(currentMapping); + newMapping.delete(fileCategory.name); + return newMapping; }); return; } @@ -249,41 +250,39 @@ export const FolderUploadComponent: FC = ({ // and set the state to 'uploadInProgress'. case 'awaitingUrls': { void (async () => { - const pendingFiles: Record = {}; - for (const [submissionId, files] of Object.entries(fileUploadState.files)) { - pendingFiles[submissionId] = await requestFileUploads(files); - } + const pendingFiles = await requestFileUploads(fileUploadState.files); setFileUploadState({ type: 'uploadInProgress', files: pendingFiles }); void startUploading(pendingFiles); })(); break; } case 'uploadInProgress': { - if ( - Object.values(fileUploadState.files) - .flatMap((x) => x) - .every(({ type }) => type === 'uploaded' || type === 'previousUpload') - ) { + if (fileUploadState.files.every(({ type }) => type === 'uploaded' || type === 'previousUpload')) { setFileUploadState({ type: 'uploadCompleted', - files: fileUploadState.files as Record, + files: fileUploadState.files as (Uploaded | PreviousUpload)[], }); } break; } case 'uploadCompleted': { - setFileMapping((currentMapping) => - produce(currentMapping ?? {}, (draft) => { - Object.entries(fileUploadState.files).forEach(([submissionId, files]) => { - if (currentMapping?.[submissionId] !== undefined) { - draft[submissionId] = { ...currentMapping[submissionId] }; - } else { - draft[submissionId] = {}; - } - draft[submissionId][fileCategory.name] = files; - }); - }), - ); + setFileMapping((currentMapping) => { + const newMapping = new Map(currentMapping); + newMapping.set( + fileCategory.name, + new Map( + fileUploadState.files.map((file) => [ + file.path, + { + name: file.name, + path: file.path, + fileId: file.fileId, + }, + ]), + ), + ); + return newMapping; + }); break; } } @@ -300,43 +299,26 @@ export const FolderUploadComponent: FC = ({ return; } - if (inputMode === 'form') { - setFileUploadState({ - type: 'awaitingUrls', - files: { - [formSubmissionId ?? DUMMY_SUBMISSION_ID]: filesArray.map((f) => ({ file: f, name: f.name })), - }, - }); - } else { - const files: Record = Object.fromEntries( - filesArray - .map((file) => file.webkitRelativePath.split('/')) - .map((pathSegments) => [pathSegments[1], []]), - ); - - filesArray.forEach((file) => { - const submissionId = file.webkitRelativePath.split('/')[1]; - files[submissionId].push({ file, name: file.name }); - }); - - setFileUploadState({ - type: 'awaitingUrls', - files, - }); - } + setFileUploadState({ + type: 'awaitingUrls', + files: filesArray.map((f) => ({ + type: 'awaiting', + file: f, + name: f.name, + path: getRelativePath(f.webkitRelativePath), + })), + }); } }; - const handleDiscardFile = (submissionId: string, file: SingleFileUpload) => { + const handleDiscardFile = (key: string) => { setFileUploadState((state) => { if (state?.type === 'uploadCompleted') { - const remainingFiles = state.files[submissionId].filter((f) => f.name !== file.name); const result = produce(state, (draft) => { - if (remainingFiles.length === 0) delete draft.files[submissionId]; - else draft.files[submissionId] = remainingFiles; + draft.files = draft.files.filter((f) => !(f.path === key)); }); - if (Object.values(result.files).every((files) => files.length === 0)) return undefined; + if (result.files.length === 0) return undefined; else return result; } return state; @@ -346,51 +328,49 @@ export const FolderUploadComponent: FC = ({ const handleDiscardAllFiles = () => setFileUploadState(undefined); const handleAddAdditionalFiles = (e: React.ChangeEvent) => { - // Currently only supported in form mode - if (inputMode === 'bulk' || !e.target.files || fileUploadState?.type !== 'uploadCompleted') return; + if (e.target.files) { + if (fileUploadState?.type !== 'uploadCompleted') return; - // exclude dot files, because files like .DS_Store cause problems otherwise - const filesArray = filterDotFiles(Array.from(e.target.files)); + // exclude dot files, because files like .DS_Store cause problems otherwise + const filesArray = filterDotFiles(Array.from(e.target.files)); - // Reset the input so selecting the same file again re-triggers onChange. - e.target.value = ''; - if (filesArray.length === 0) return; + // Reset the input so selecting the same file again re-triggers onChange. + e.target.value = ''; + if (filesArray.length === 0) return; - // Check for duplicate file names in the selection - const selectedFileNames = filesArray.map((f) => f.name); - const duplicateName = selectedFileNames.find((name, index) => selectedFileNames.indexOf(name) !== index); - if (duplicateName !== undefined) { - onError(`Cannot add multiple files with the same name: ${duplicateName}`); - return; - } + const awaiting: Awaiting[] = filesArray.map((f) => ({ + type: 'awaiting', + file: f, + name: f.name, + path: f.name, + })); - // Check for collisions with existing files - const submissionId = formSubmissionId ?? DUMMY_SUBMISSION_ID; - const existingFiles = fileUploadState.files[submissionId]; - const uniqueSelectedNames = new Set(selectedFileNames); - const existingFileCollisions = existingFiles.filter((file) => uniqueSelectedNames.has(file.name)); + // Check for collisions with existing files + const filePaths = new Set(awaiting.map((file) => file.path)); + const collisions = fileUploadState.files.filter((file) => filePaths.has(file.path)); - // Updates the state of file uploads and triggers the upload of the new files - const addAdditionalFiles = async () => { - const nonCollidingFiles = existingFiles.filter((file) => !uniqueSelectedNames.has(file.name)); - const newPendingFiles = await requestFileUploads(filesArray.map((f) => ({ file: f, name: f.name }))); - setFileUploadState({ - type: 'uploadInProgress', - files: { [submissionId]: [...nonCollidingFiles, ...newPendingFiles] }, - }); - void startUploading({ [submissionId]: newPendingFiles }); - }; - - // If there are collisions, show a confirmation dialog before proceeding - if (existingFileCollisions.length > 0) { - displayConfirmationDialog({ - dialogText: - 'The following file(s) already exist and will be replaced: ' + - existingFileCollisions.map((file) => file.name).join(', '), - confirmButtonText: 'Replace', - onConfirmation: addAdditionalFiles, - }); - } else void addAdditionalFiles(); + // Updates the state of file uploads and triggers the upload of the new files + const addAdditionalFiles = async () => { + const existingFiles = fileUploadState.files.filter((file) => !filePaths.has(file.path)); + const pendingFiles = await requestFileUploads(awaiting); + setFileUploadState({ + type: 'uploadInProgress', + files: [...existingFiles, ...pendingFiles], + }); + void startUploading(pendingFiles); + }; + + // If there are collisions, show a confirmation dialog before proceeding + if (collisions.length > 0) { + displayConfirmationDialog({ + dialogText: + 'The following file(s) already exist and will be replaced: ' + + collisions.map((file) => file.name).join(', '), + confirmButtonText: 'Replace', + onConfirmation: addAdditionalFiles, + }); + } else void addAdditionalFiles(); + } }; return fileUploadState === undefined || fileUploadState.type === 'awaitingUrls' ? ( @@ -461,61 +441,48 @@ export const FolderUploadComponent: FC = ({
-

Files

- {inputMode === 'form' - ? fileUploadState.files[formSubmissionId ?? DUMMY_SUBMISSION_ID].map((file) => ( -
-
- -
- -
- )) - : Object.entries(fileUploadState.files).flatMap(([submissionId, files]) => [ -

- {submissionId} -

, - ...files.map((file) => ), - ])} -
    +

    {fileCategory.displayName ?? fileCategory.name}

    + {fileUploadState.files.map((file) => ( +
    +
    + +
    + +
    + ))}
    - -
    - {inputMode === 'form' && ( - <> - {isClient && ( - - )} - - +
    + {isClient && ( + )} + diff --git a/website/src/components/Submission/FormOrUploadWrapper.tsx b/website/src/components/Submission/FormOrUploadWrapper.tsx index d2550a230b..f701b4a7ab 100644 --- a/website/src/components/Submission/FormOrUploadWrapper.tsx +++ b/website/src/components/Submission/FormOrUploadWrapper.tsx @@ -8,6 +8,7 @@ import type { InputField, SubmissionDataTypes } from '../../types/config'; import { EditableSequences } from '../Edit/EditableSequences'; import { EditableMetadata, MetadataForm } from '../Edit/MetadataForm'; import { SequencesForm } from '../Edit/SequencesForm'; +import { parseSubmissionFileMapping, type SubmissionFileMapping } from './FileUpload/fileMapping'; export type InputMode = 'form' | 'bulk'; @@ -39,10 +40,12 @@ export type FileFactory = () => Promise; type FormOrUploadWrapperProps = { inputMode: InputMode; setFileFactory: Dispatch>; + setSubmissionFileMapping: Dispatch>; organism: string; action: UploadAction; metadataTemplateFields: Map; submissionDataTypes: SubmissionDataTypes; + onError: (message: string) => void; }; /** @@ -55,10 +58,12 @@ type FormOrUploadWrapperProps = { export const FormOrUploadWrapper: FC = ({ inputMode, setFileFactory, + setSubmissionFileMapping, organism, action, metadataTemplateFields, submissionDataTypes, + onError, }) => { const enableConsensusSequences = submissionDataTypes.consensusSequences; const [editableMetadata, setEditableMetadata] = useState(EditableMetadata.empty()); @@ -71,6 +76,28 @@ export const FormOrUploadWrapper: FC = ({ // The columnMapping can be null; if null -> don't apply mapping. const [columnMapping, setColumnMapping] = useState(null); + useEffect(() => { + const controller = new AbortController(); + void (async () => { + if (!metadataFile) { + setSubmissionFileMapping(undefined); + return; + } + const text = columnMapping + ? await (await columnMapping.applyTo(metadataFile)).text() + : await metadataFile.text(); + if (!controller.signal.aborted) { + const submissionFileMapping = parseSubmissionFileMapping(text); + if (submissionFileMapping.isOk()) setSubmissionFileMapping(submissionFileMapping.value); + else { + setSubmissionFileMapping(undefined); + onError(submissionFileMapping.error.message); + } + } + })(); + return () => controller.abort(); + }, [metadataFile, columnMapping]); + useEffect(() => { setFileFactory(() => { // Returns a function that the parent component can call to get the files needed for submission