From 718e38077c53735d00372b2e9c4e0a5733f08404 Mon Sep 17 00:00:00 2001 From: James Noss Date: Wed, 29 May 2024 21:21:22 -0400 Subject: [PATCH 1/5] Reduce user migrations to 0001 Signed-off-by: James Noss --- biodb/apps/user/migrations/0001_initial.py | 7 ++--- ...sqluser_user_is_sqluser_change_and_more.py | 27 ------------------- .../user/migrations/0003_alter_user_center.py | 19 ------------- 3 files changed, 4 insertions(+), 49 deletions(-) delete mode 100644 biodb/apps/user/migrations/0002_remove_user_is_sqluser_user_is_sqluser_change_and_more.py delete mode 100644 biodb/apps/user/migrations/0003_alter_user_center.py diff --git a/biodb/apps/user/migrations/0001_initial.py b/biodb/apps/user/migrations/0001_initial.py index 85b781a..7cc52c0 100644 --- a/biodb/apps/user/migrations/0001_initial.py +++ b/biodb/apps/user/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.7 on 2023-12-08 21:03 +# Generated by Django 4.2.11 on 2024-05-30 01:20 import django.contrib.auth.validators from django.db import migrations, models @@ -43,9 +43,10 @@ class Migration(migrations.Migration): ('is_staff', models.BooleanField(default=True, help_text='Designates whether the user can log into this admin site.', verbose_name='staff status')), ('is_active', models.BooleanField(default=True, help_text='Designates whether this user should be treated as active. Unselect this instead of deleting accounts.', verbose_name='active')), ('date_joined', models.DateTimeField(default=django.utils.timezone.now, verbose_name='date joined')), - ('is_sqluser', models.BooleanField(default=False, help_text='Designates whether the user can log into the SQL explorer app.', verbose_name='SQL explorer user status')), + ('is_sqluser_view', models.BooleanField(default=False, help_text='Designates whether the user can log into the SQL explorer app with permissions to only view and execute existing queries.', verbose_name='SQL explorer user status (view/execute existing queries only)')), + ('is_sqluser_change', models.BooleanField(default=False, help_text='Designates whether the user can log into the SQL explorer app with permissions to view/add/change/delete/execute queries.', verbose_name='SQL explorer user status (view/add/change/delete/execute)')), ('is_catalogviewer', models.BooleanField(default=False, help_text='Designates whether the user can log into the Dataset Catalog app. (readonly)', verbose_name='Dataset Catalog user status (readonly)')), - ('center', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='user', to='user.center')), + ('center', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='user', to='user.center')), ('groups', models.ManyToManyField(blank=True, help_text='The groups this user belongs to. A user will get all permissions granted to each of their groups.', related_name='user_set', related_query_name='user', to='auth.group', verbose_name='groups')), ('user_permissions', models.ManyToManyField(blank=True, help_text='Specific permissions for this user.', related_name='user_set', related_query_name='user', to='auth.permission', verbose_name='user permissions')), ], diff --git a/biodb/apps/user/migrations/0002_remove_user_is_sqluser_user_is_sqluser_change_and_more.py b/biodb/apps/user/migrations/0002_remove_user_is_sqluser_user_is_sqluser_change_and_more.py deleted file mode 100644 index db0e88d..0000000 --- a/biodb/apps/user/migrations/0002_remove_user_is_sqluser_user_is_sqluser_change_and_more.py +++ /dev/null @@ -1,27 +0,0 @@ -# Generated by Django 4.2.11 on 2024-03-16 12:56 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('user', '0001_initial'), - ] - - operations = [ - migrations.RemoveField( - model_name='user', - name='is_sqluser', - ), - migrations.AddField( - model_name='user', - name='is_sqluser_change', - field=models.BooleanField(default=False, help_text='Designates whether the user can log into the SQL explorer app with permissions to view/add/change/delete/execute queries.', verbose_name='SQL explorer user status (view/add/change/delete/execute)'), - ), - migrations.AddField( - model_name='user', - name='is_sqluser_view', - field=models.BooleanField(default=False, help_text='Designates whether the user can log into the SQL explorer app with permissions to only view and execute existing queries.', verbose_name='SQL explorer user status (view/execute existing queries only)'), - ), - ] diff --git a/biodb/apps/user/migrations/0003_alter_user_center.py b/biodb/apps/user/migrations/0003_alter_user_center.py deleted file mode 100644 index 8f570ac..0000000 --- a/biodb/apps/user/migrations/0003_alter_user_center.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 4.2.11 on 2024-03-20 14:24 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('user', '0002_remove_user_is_sqluser_user_is_sqluser_change_and_more'), - ] - - operations = [ - migrations.AlterField( - model_name='user', - name='center', - field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='user', to='user.center'), - ), - ] From 0b48a7a4fde63d77b543a773f8b53a5f2cf217a2 Mon Sep 17 00:00:00 2001 From: James Noss Date: Wed, 29 May 2024 21:50:53 -0400 Subject: [PATCH 2/5] SpctralData -> ArrayData (and reduce migrations to 0001) Signed-off-by: James Noss --- biodb/apps/catalog/admin.py | 18 +- biodb/apps/catalog/migrations/0001_initial.py | 4 +- biodb/apps/catalog/models.py | 16 +- biodb/apps/catalog/tests/test_datasets.py | 4 +- biodb/apps/uploader/admin.py | 42 ++--- biodb/apps/uploader/base_models.py | 2 +- biodb/apps/uploader/charts.py | 18 +- biodb/apps/uploader/exporters.py | 32 ++-- biodb/apps/uploader/io.py | 42 ++--- biodb/apps/uploader/loaddata.py | 44 ++--- .../management/commands/run_qc_annotators.py | 12 +- .../apps/uploader/migrations/0001_initial.py | 168 ++++++++--------- .../0002_alter_patient_patient_cid.py | 18 -- ..._alter_biosample_freezing_time_and_more.py | 23 --- .../0004_alter_biosample_thawing_time.py | 18 -- ...lter_biosample_centrifuge_time_and_more.py | 43 ----- biodb/apps/uploader/models.py | 173 +++++++----------- biodb/apps/uploader/tests/conftest.py | 16 +- biodb/apps/uploader/tests/test_charts.py | 2 +- biodb/apps/uploader/tests/test_exporters.py | 28 +-- biodb/apps/uploader/tests/test_io.py | 38 ++-- .../tests/test_management_commands.py | 10 +- biodb/apps/uploader/tests/test_models.py | 88 ++++----- biodb/apps/uploader/tests/test_qc_models.py | 26 +-- .../uploader/tests/test_uploader_admin.py | 10 +- biodb/qc/qcfilter.py | 12 +- biodb/qc/qcmanager.py | 4 +- biodb/settings/base.py | 8 +- biodb/util.py | 6 +- 29 files changed, 380 insertions(+), 545 deletions(-) delete mode 100644 biodb/apps/uploader/migrations/0002_alter_patient_patient_cid.py delete mode 100644 biodb/apps/uploader/migrations/0003_alter_biosample_freezing_time_and_more.py delete mode 100644 biodb/apps/uploader/migrations/0004_alter_biosample_thawing_time.py delete mode 100644 biodb/apps/uploader/migrations/0005_alter_biosample_centrifuge_time_and_more.py diff --git a/biodb/apps/catalog/admin.py b/biodb/apps/catalog/admin.py index 887a9e5..501a77a 100644 --- a/biodb/apps/catalog/admin.py +++ b/biodb/apps/catalog/admin.py @@ -27,7 +27,7 @@ def has_delete_permission(self, request, obj=None): @admin.register(Dataset) class DatasetAdmin(AuthMixin, admin.ModelAdmin): search_fields = ["created_at", "name", "version", "query__name"] - list_display = ["name", "version", "file", "created_at", "size", "n_rows", "n_spectral_data_files"] + list_display = ["name", "version", "file", "created_at", "size", "n_rows", "n_array_data_files"] date_hierarchy = "created_at" ordering = ("-updated_at",) list_filter = ("name",) @@ -40,8 +40,8 @@ class DatasetAdmin(AuthMixin, admin.ModelAdmin): "updated_at", "id", "n_rows", - "n_spectral_data_files", - "spectral_data_filenames", + "n_array_data_files", + "array_data_filenames", "data_sha256"] fieldsets = [ @@ -73,14 +73,14 @@ class DatasetAdmin(AuthMixin, admin.ModelAdmin): "app_version", "id", "n_rows", - "n_spectral_data_files"], + "n_array_data_files"], } ), ( - "Spectral Data Filenames", + "Array Data Filenames", { "classes": ["collapse"], - "fields": ["spectral_data_filenames"], + "fields": ["array_data_filenames"], } ), ] @@ -91,9 +91,9 @@ def size(self, obj): return f"{int(obj.file.size / 1e6)} MB" @admin.display - def n_spectral_data_files(self, obj): - if obj.spectral_data_filenames: - return len(obj.spectral_data_filenames) + def n_array_data_files(self, obj): + if obj.array_data_filenames: + return len(obj.array_data_filenames) return 0 diff --git a/biodb/apps/catalog/migrations/0001_initial.py b/biodb/apps/catalog/migrations/0001_initial.py index 607b6e1..8342c78 100644 --- a/biodb/apps/catalog/migrations/0001_initial.py +++ b/biodb/apps/catalog/migrations/0001_initial.py @@ -30,8 +30,8 @@ class Migration(migrations.Migration): ('app_version', models.CharField(blank=True, default=catalog.models.get_app_version, editable=False, help_text='App version used to create data product', max_length=32)), ('sha256', models.CharField(blank=True, editable=False, help_text='Checksum of downloadable file', max_length=64, validators=[django.core.validators.MinLengthValidator(64)], verbose_name='SHA-256')), ('n_rows', models.IntegerField(blank=True, editable=False, help_text='Number of data rows')), - ('data_sha256', models.CharField(blank=True, editable=False, help_text='Checksum of data table (not including any spectral data files).', max_length=64, validators=[django.core.validators.MinLengthValidator(64)], verbose_name='Data SHA-256')), - ('spectral_data_filenames', models.JSONField(blank=True, default=catalog.models.empty_list, editable=False, encoder=catalog.models.CustomDjangoJsonEncoder, help_text='List of spectral data filenames')), + ('data_sha256', models.CharField(blank=True, editable=False, help_text='Checksum of data table (not including any array data files).', max_length=64, validators=[django.core.validators.MinLengthValidator(64)], verbose_name='Data SHA-256')), + ('array_data_filenames', models.JSONField(blank=True, default=catalog.models.empty_list, editable=False, encoder=catalog.models.CustomDjangoJsonEncoder, help_text='List of array data filenames')), ('query', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='dataset', to='explorer.query')), ], options={ diff --git a/biodb/apps/catalog/models.py b/biodb/apps/catalog/models.py index 6df8217..032a145 100644 --- a/biodb/apps/catalog/models.py +++ b/biodb/apps/catalog/models.py @@ -46,7 +46,7 @@ class Dataset(DatedModel): sha256 (:obj:`django.models.CharField`): The SHA-256 checksum of the entire zip file. n_rows (:obj:`django.models.IntegerField`): The number of rows in the zipped data file. Depending on the query, this could be the total number of patients or something else. data_sha256 (:obj:`django.models.IntegerField`): The SHA-256 checksum of the data file archived within the zip file. - spectral_data_filenames (:obj:`django.models.JSONField`): A list of all the file names for all individual spectral data files zipped within the downloadable zip file. + array_data_filenames (:obj:`django.models.JSONField`): A list of all the file names for all individual array data files zipped within the downloadable zip file. """ class Meta: @@ -99,13 +99,13 @@ class Meta: null=False, blank=True, verbose_name="Data SHA-256", - help_text="Checksum of data table (not including any spectral data files).", + help_text="Checksum of data table (not including any array data files).", validators=[MinLengthValidator(64)]) - spectral_data_filenames = models.JSONField(null=False, + array_data_filenames = models.JSONField(null=False, default=empty_list, blank=True, editable=False, - help_text="List of spectral data filenames", + help_text="List of array data filenames", encoder=CustomDjangoJsonEncoder) def __str__(self): @@ -124,7 +124,7 @@ def clean(self, *args, **kwargs): if not self.file: # Create file from query. file, info = self.execute_query() - filename, n_rows, data_sha256, spectral_data_filenames = info + filename, n_rows, data_sha256, array_data_filenames = info if not n_rows: raise ValidationError(_("Query returned no data.")) @@ -133,7 +133,7 @@ def clean(self, *args, **kwargs): self._filename = filename self.n_rows = n_rows self.data_sha256 = data_sha256 - self.spectral_data_filenames = spectral_data_filenames + self.array_data_filenames = array_data_filenames super().clean(*args, **kwargs) @@ -182,9 +182,9 @@ def meta_info(self, **kwargs): app_version=self.app_version, id=str(self.id), n_rows=self.n_rows, - n_spectral_data_files=len(self.spectral_data_filenames), + n_array_data_files=len(self.array_data_filenames), timestamp=str(datetime.datetime.now()), - spectral_data_filenames=self.spectral_data_filenames) + array_data_filenames=self.array_data_filenames) info.update(kwargs) return info diff --git a/biodb/apps/catalog/tests/test_datasets.py b/biodb/apps/catalog/tests/test_datasets.py index 338de73..792fa0b 100644 --- a/biodb/apps/catalog/tests/test_datasets.py +++ b/biodb/apps/catalog/tests/test_datasets.py @@ -8,7 +8,7 @@ from biodb import __version__ from catalog.models import Dataset from explorer.models import Query -from uploader.models import SpectralData +from uploader.models import ArrayData @pytest.mark.django_db(databases=["default", "bsr"]) @@ -49,7 +49,7 @@ def test_files(self, saved_dataset): namelist = z.namelist() namelist.remove(str(Path(saved_dataset.name).with_suffix(file_ext))) namelist.remove("INFO.json") - data_dir = Path(SpectralData.UPLOAD_DIR) + data_dir = Path(ArrayData.UPLOAD_DIR) for file in namelist: assert Path(file).parent == data_dir diff --git a/biodb/apps/uploader/admin.py b/biodb/apps/uploader/admin.py index 1dfadde..4264d9c 100644 --- a/biodb/apps/uploader/admin.py +++ b/biodb/apps/uploader/admin.py @@ -11,8 +11,8 @@ from nested_admin import NestedStackedInline, NestedTabularInline, NestedModelAdmin from biodb.util import to_bool -from .models import BioSample, Observable, Instrument, Patient, SpectralData, Observation, UploadedFile, Visit,\ - QCAnnotator, QCAnnotation, Center, get_center, BioSampleType, SpectraMeasurementType +from .models import BioSample, Observable, Instrument, Patient, ArrayData, Observation, UploadedFile, Visit,\ + QCAnnotator, QCAnnotation, Center, get_center, BioSampleType, ArrayMeasurementType from uploader.forms import ModelForm from user.admin import CenterAdmin as UserCenterAdmin @@ -127,10 +127,10 @@ def formfield_for_foreignkey(self, db_field, request, **kwargs): field.queryset = field.queryset.filter(visit__center=center) elif db_field.name == "bio_sample": field.queryset = field.queryset.filter(visit__patient__center=center) - elif db_field.name == "spectral_data": + elif db_field.name == "array_data": field.queryset = field.queryset.filter(bio_sample__visit__patient__center=center) elif db_field.name == "qc_annotation": - field.queryset = field.queryset.filter(spectral_data__bio_sample__visit__patient__center=center) + field.queryset = field.queryset.filter(array_data__bio_sample__visit__patient__center=center) elif db_field.name in ("annotator", "measurement_type", "sample_type"): # These aren't limited/restricted by center. pass @@ -142,7 +142,7 @@ def formfield_for_foreignkey(self, db_field, request, **kwargs): admin.site.register(BioSampleType) -admin.site.register(SpectraMeasurementType) +admin.site.register(ArrayMeasurementType) @admin.register(Instrument) @@ -218,7 +218,7 @@ class UploadedFileAdmin(RestrictedByCenterMixin, ModelAdmin): form = UploadedFileForm search_fields = ["created_at"] search_help_text = "Creation timestamp" - list_display = ["pk", "created_at", "meta_data_file", "spectral_data_file", "center"] + list_display = ["pk", "created_at", "meta_data_file", "array_data_file", "center"] readonly_fields = ["created_at", "updated_at"] # TODO: Might need specific user group. date_hierarchy = "created_at" ordering = ("-updated_at",) @@ -246,13 +246,13 @@ def get_extra(self, request, obj=None, **kwargs): @admin.register(QCAnnotation) class QCAnnotationAdmin(RestrictedByCenterMixin, ModelAdmin): search_fields = ["annotator__name", - "spectral_data__bio_sample__visit__patient__patient_id", - "spectral_data__bio_sample__visit__patient__patient_cid"] + "array_data__bio_sample__visit__patient__patient_id", + "array_data__bio_sample__visit__patient__patient_cid"] search_help_text = "Annotator Name, Patient ID or CID" readonly_fields = ("value", "created_at", "updated_at") # TODO: Might need specific user group for timestamps. list_display = ["annotator_name", "value", "annotator_value_type", "updated_at"] ordering = ("-updated_at",) - list_filter = ("spectral_data__bio_sample__visit__patient__center", "annotator__name") + list_filter = ("array_data__bio_sample__visit__patient__center", "annotator__name") @admin.display def annotator_name(self, obj): @@ -268,7 +268,7 @@ def get_queryset(self, request): if request.user.is_superuser: return qs center = Center.objects.get(pk=request.user.center.pk) - return qs.filter(spectral_data__bio_sample__visit__patient__center=center) + return qs.filter(array_data__bio_sample__visit__patient__center=center) @admin.register(QCAnnotator) @@ -452,7 +452,7 @@ class ObservationAdmin(ObservationMixin, RestrictedByCenterMixin, NestedModelAdm list_display = ["patient_id", "observable_name", "visit"] -class SpectralDataMixin: +class ArrayDataMixin: ordering = ("-updated_at",) readonly_fields = ["created_at", "updated_at"] @@ -510,8 +510,8 @@ def get_queryset(self, request): return qs.filter(bio_sample__visit__patient__center=Center.objects.get(pk=request.user.center.pk)) -@admin.register(SpectralData) -class SpectralDataAdmin(SpectralDataMixin, RestrictedByCenterMixin, NestedModelAdmin): +@admin.register(ArrayData) +class ArrayDataAdmin(ArrayDataMixin, RestrictedByCenterMixin, NestedModelAdmin): search_fields = ["bio_sample__visit__patient__patient_id", "bio_sample__visit__patient__patient_cid"] search_help_text = "Patient ID or CID" readonly_fields = ["created_at", "updated_at"] # TODO: Might need specific user group. @@ -525,12 +525,12 @@ class SpectralDataAdmin(SpectralDataMixin, RestrictedByCenterMixin, NestedModelA "bio_sample__visit__observation__observable") -class SpectralDataAdminWithInlines(SpectralDataAdmin): +class ArrayDataAdminWithInlines(ArrayDataAdmin): inlines = [QCAnnotationInline] -class SpectralDataInline(SpectralDataMixin, RestrictedByCenterMixin, NestedStackedInline): - model = SpectralData +class ArrayDataInline(ArrayDataMixin, RestrictedByCenterMixin, NestedStackedInline): + model = ArrayData extra = 1 min_num = 0 show_change_link = True @@ -538,7 +538,7 @@ class SpectralDataInline(SpectralDataMixin, RestrictedByCenterMixin, NestedStack def get_extra(self, request, obj=None, **kwargs): # Only display inlines for those that exist, i.e., no expanded extras (if they exist). - return 0 if obj and obj.pk and obj.spectral_data.count() else self.extra + return 0 if obj and obj.pk and obj.array_data.count() else self.extra class BioSampleMixin: @@ -600,7 +600,7 @@ class BioSampleAdmin(BioSampleMixin, RestrictedByCenterMixin, NestedModelAdmin): class BioSampleAdminWithInlines(BioSampleAdmin): - inlines = [SpectralDataInline] + inlines = [ArrayDataInline] class BioSampleInline(BioSampleMixin, RestrictedByCenterMixin, NestedStackedInline): @@ -609,7 +609,7 @@ class BioSampleInline(BioSampleMixin, RestrictedByCenterMixin, NestedStackedInli min_num = 0 show_change_link = True fk_name = "visit" - inlines = [SpectralDataInline] + inlines = [ArrayDataInline] def get_extra(self, request, obj=None, **kwargs): # Only display inlines for those that exist, i.e., no expanded extras (if they exist). @@ -796,7 +796,7 @@ class DataAdminSite(admin.AdminSite): Visit, Observation, BioSample, - SpectralData, + ArrayData, UploadedFile ] @@ -815,7 +815,7 @@ def get_app_list(self, request, app_label=None): data_admin.register(Visit, admin_class=VisitAdminWithInlines) data_admin.register(Observation, admin_class=ObservationAdmin) data_admin.register(BioSample, admin_class=BioSampleAdminWithInlines) -data_admin.register(SpectralData, admin_class=SpectralDataAdminWithInlines) +data_admin.register(ArrayData, admin_class=ArrayDataAdminWithInlines) data_admin.register(UploadedFile, admin_class=UploadedFileAdmin) # data_admin.register(Instrument, admin_class=InstrumentAdmin) # data_admin.register(QCAnnotation, admin_class=QCAnnotationAdmin) diff --git a/biodb/apps/uploader/base_models.py b/biodb/apps/uploader/base_models.py index 56d0ced..6ed60e3 100644 --- a/biodb/apps/uploader/base_models.py +++ b/biodb/apps/uploader/base_models.py @@ -20,7 +20,7 @@ def get_column_names(cls, help_text=False): "data", "date", "id", - "spectral data file", + "array data file", "updated at"} if hasattr(cls, "parse_fields_from_pandas_series"): # Only models with this func have bulk data upload columns. diff --git a/biodb/apps/uploader/charts.py b/biodb/apps/uploader/charts.py index c7d2c1e..546ebc4 100644 --- a/biodb/apps/uploader/charts.py +++ b/biodb/apps/uploader/charts.py @@ -7,7 +7,7 @@ import plotly.graph_objects as go import uploader.io -from uploader.models import Observable, Patient, SpectralData +from uploader.models import Observable, Patient, ArrayData from biodb.util import to_uuid logger = logging.getLogger(__name__) @@ -54,7 +54,7 @@ def get_pie_chart(result: "QueryResult") -> Optional[str]: # noqa: F821 def get_line_chart(result: "QueryResult") -> Optional[str]: # noqa: F821 - """ Generate ploty line chart of spectral data present in data result. """ + """ Generate ploty line chart of array data present in data result. """ if len(result.data) < 1: return @@ -62,19 +62,19 @@ def get_line_chart(result: "QueryResult") -> Optional[str]: # noqa: F821 try: df = pd.DataFrame(result.data, columns=result.header_strings) - if Patient.patient_id.field.name not in df.columns or SpectralData.data.field.name not in df.columns: + if Patient.patient_id.field.name not in df.columns or ArrayData.data.field.name not in df.columns: return - df = df[[Patient.patient_id.field.name, SpectralData.data.field.name]] + df = df[[Patient.patient_id.field.name, ArrayData.data.field.name]] fig = go.Figure() fig.update_layout(xaxis_title="Wavelength", yaxis_title="Intensity", - title=f"Spectral Data for SQL query: '{result.sql}'") + title=f"Array Data for SQL query: '{result.sql}'") for row in df.itertuples(): - spectral_data = uploader.io.read_spectral_data(row.data) - assert to_uuid(spectral_data.patient_id) == to_uuid(row.patient_id) - fig.add_scatter(x=spectral_data.wavelength, - y=spectral_data.intensity, + array_data = uploader.io.read_array_data(row.data) + assert to_uuid(array_data.patient_id) == to_uuid(row.patient_id) + fig.add_scatter(x=array_data.wavelength, + y=array_data.intensity, name=str(row.patient_id)) return fig_to_html(fig) diff --git a/biodb/apps/uploader/exporters.py b/biodb/apps/uploader/exporters.py index bc01a66..fda7e73 100644 --- a/biodb/apps/uploader/exporters.py +++ b/biodb/apps/uploader/exporters.py @@ -7,11 +7,11 @@ import explorer.exporters import pandas as pd -from uploader.models import SpectralData +from uploader.models import ArrayData -class ZipSpectralDataMixin: - """ A custom mixin for explorer.exporters.BaseExporter used to collect SpectralData.data files and zip them with +class ZipArrayDataMixin: + """ A custom mixin for explorer.exporters.BaseExporter used to collect ArrayData.data files and zip them with query output data for download. """ @@ -61,7 +61,7 @@ def get_file_output(self, output = self._get_output(res, **kwargs) n_rows = len(res.data) - spectral_data_filenames = None + array_data_filenames = None # Compute data checksum output.seek(0) @@ -73,20 +73,20 @@ def get_file_output(self, data_files = [] if include_data_files: - storage = SpectralData.data.field.storage - # Collect SpectralData files and zip along with query data from self._get_output(). + storage = ArrayData.data.field.storage + # Collect ArrayData files and zip along with query data from self._get_output(). if settings.EXPLORER_DATA_EXPORTERS_ALLOW_DATA_FILE_ALIAS: - # Spectral data files are modeled by the Spectraldata.data field, however, the sql query could have + # Array data files are modeled by the Arraydata.data field, however, the sql query could have # aliased these so it wouldn't be safe to search by column name. Instead, we can only exhaustively - # search all entries for some marker indicating that they are spectral data files, where this "marker" - # is the upload directory - SpectralData.data.field.upload_to. - upload_dir = SpectralData.data.field.upload_to # NOTE: We don't need to inc. the MEDIA_ROOT for this. + # search all entries for some marker indicating that they are array data files, where this "marker" + # is the upload directory - ArrayData.data.field.upload_to. + upload_dir = ArrayData.data.field.upload_to # NOTE: We don't need to inc. the MEDIA_ROOT for this. for row in res.data: for item in row: if isinstance(item, str) and item.startswith(upload_dir): data_files.append(item) else: - if (col_name := SpectralData.data.field.name) in res.header_strings: + if (col_name := ArrayData.data.field.name) in res.header_strings: df = pd.DataFrame(res.data, columns=res.header_strings) df = df[col_name] # There could be multiple "col_name" (aka "data") columns so flatten first. @@ -95,7 +95,7 @@ def get_file_output(self, if data_files or always_zip: # Dedupe and sort. data_files = sorted(set(data_files)) - spectral_data_filenames = data_files + array_data_filenames = data_files # Zip everything together. temp = tempfile.TemporaryFile() @@ -120,16 +120,16 @@ def get_file_output(self, output = temp self.is_zip = True - return (output, (n_rows, data_sha256, spectral_data_filenames)) if return_info else output + return (output, (n_rows, data_sha256, array_data_filenames)) if return_info else output -class CSVExporter(ZipSpectralDataMixin, explorer.exporters.CSVExporter): +class CSVExporter(ZipArrayDataMixin, explorer.exporters.CSVExporter): ... -class ExcelExporter(ZipSpectralDataMixin, explorer.exporters.ExcelExporter): +class ExcelExporter(ZipArrayDataMixin, explorer.exporters.ExcelExporter): ... -class JSONExporter(ZipSpectralDataMixin, explorer.exporters.JSONExporter): +class JSONExporter(ZipArrayDataMixin, explorer.exporters.JSONExporter): ... diff --git a/biodb/apps/uploader/io.py b/biodb/apps/uploader/io.py index e6c2b2a..6f25092 100644 --- a/biodb/apps/uploader/io.py +++ b/biodb/apps/uploader/io.py @@ -22,7 +22,7 @@ @dataclasses.dataclass -class SpectralData: +class ArrayData: patient_id: UUID wavelength: list intensity: list @@ -31,7 +31,7 @@ def __post_init__(self): self.patient_id = to_uuid(self.patient_id) def to_json(self, filename, **kwargs): - return spectral_data_to_json(filename, data=self, **kwargs) + return array_data_to_json(filename, data=self, **kwargs) JSON_OPTS = {"indent": None, "force_ascii": True} # jsonlines. @@ -143,7 +143,7 @@ def read_meta_data(file, index_column=settings.BULK_UPLOAD_INDEX_COLUMN_NAME): return cleaned_data -def read_spectral_data_table(file, index_column=DEFAULT_PATIENT_ID_STR): +def read_array_data_table(file, index_column=DEFAULT_PATIENT_ID_STR): """ Read in multiple rows of data returning a pandas.DataFrame. The data to be read in, needs to be of the following table layout for .csv & .xlsx: @@ -156,7 +156,7 @@ def read_spectral_data_table(file, index_column=DEFAULT_PATIENT_ID_STR): For .jsonl each line/row must be of the following: {"patient_id": value, wavelength_value: intensity_value, wavelength_value: intensity_value, ...} - For json data of the following form use ``read_spectral_data()``. + For json data of the following form use ``read_array_data()``. {"patient_id": value, "wavelength": [values], "intensity": [values]} """ df = _read_raw_data(file) @@ -175,13 +175,13 @@ def read_spectral_data_table(file, index_column=DEFAULT_PATIENT_ID_STR): return df -def read_single_row_spectral_data_table(file, index_column=DEFAULT_PATIENT_ID_STR): - """ Read in single row spectral data. +def read_single_row_array_data_table(file, index_column=DEFAULT_PATIENT_ID_STR): + """ Read in single row array data. The data to be read in, needs to be of the following table layout for .csv & .xlsx: Note: Commas need to be present for CSV data. Note: The following docstring uses markdown table syntax. - Note: This is as for ``read_spectral_data_table`` except that it contains data for only a single + Note: This is as for ``read_array_data_table`` except that it contains data for only a single patient, i.e., just a single row: | patient_id | min_lambda | ... | max_lambda | | ---------- | ---------- | --- | ---------- | @@ -190,20 +190,20 @@ def read_single_row_spectral_data_table(file, index_column=DEFAULT_PATIENT_ID_ST For .jsonl each line/row must be: {"patient_id": value, wavelength_value: intensity_values, wavelength_value: intensity_values, ...} - For json data of the following form use ``read_spectral_data()``. + For json data of the following form use ``read_array_data()``. {"patient_id": value, "wavelength": [values], "intensity": [values]} """ - df = read_spectral_data_table(file, index_column=index_column) + df = read_array_data_table(file, index_column=index_column) if (length := len(df)) != 1: raise ValueError(f"The file read should contain only a single row not '{length}'") data = df.iloc[0] - return SpectralData(data.patient_id, data.wavelength, data.intensity) + return ArrayData(data.patient_id, data.wavelength, data.intensity) -def spectral_data_to_json(file, data: SpectralData, patient_id=None, wavelength=None, intensity=None, **kwargs): +def array_data_to_json(file, data: ArrayData, patient_id=None, wavelength=None, intensity=None, **kwargs): """ Convert data to json equivalent to {"patient_id": value, "wavelength": [values], "intensity": [values]} Returns json str and/or writes to file. """ @@ -214,7 +214,7 @@ def spectral_data_to_json(file, data: SpectralData, patient_id=None, wavelength= assert intensity is not None data = dict(patient_id=patient_id, wavelength=wavelength, intensity=intensity) - if isinstance(data, SpectralData): + if isinstance(data, ArrayData): data = dataclasses.asdict(data) opts = dict(indent=JSON_OPTS["indent"], ensure_ascii=JSON_OPTS["force_ascii"], cls=DjangoJSONEncoder) @@ -229,9 +229,9 @@ def spectral_data_to_json(file, data: SpectralData, patient_id=None, wavelength= json.dump(data, file, **opts) -def spectral_data_from_json(file): - """ Read spectral data of the form {"patient_id": value, "wavelength": [values], "intensity": [values]} - and return data SpectralData instance. +def array_data_from_json(file): + """ Read array data of the form {"patient_id": value, "wavelength": [values], "intensity": [values]} + and return data ArrayData instance. """ # Determine whether file obj (fp) or filename. fp, filename = _get_file_info(file) @@ -250,19 +250,19 @@ def spectral_data_from_json(file): raise ValueError("A path-like or file-like object must be specified.") # Check that the json is as expected. This is needed for validation when a user provides json data. - if (fields := {x.name for x in dataclasses.fields(SpectralData)}) != data.keys(): + if (fields := {x.name for x in dataclasses.fields(ArrayData)}) != data.keys(): raise DataSchemaError(f"Schema error: expected only the fields '{fields}' but got '{data.keys()}'") - return SpectralData(**data) + return ArrayData(**data) -def read_spectral_data(file): - """ General purpose reader to handle multiple file formats returning SpectralData instance. +def read_array_data(file): + """ General purpose reader to handle multiple file formats returning ArrayData instance. The data to be read in, needs to be of the following table layout for .csv & .xlsx: Note: Commas need to be present for CSV data. Note: The following docstring uses markdown table syntax. - Note: This is as for ``read_spectral_data_table`` except that it contains data for only a single + Note: This is as for ``read_array_data_table`` except that it contains data for only a single patient, i.e., just a single row: | patient_id | min_lambda | ... | max_lambda | | ---------- | ---------- | --- | ---------- | @@ -274,7 +274,7 @@ def read_spectral_data(file): _fp, filename = _get_file_info(file) ext = filename.suffix - data = spectral_data_from_json(file) if ext == FileFormats.JSONL else read_single_row_spectral_data_table(file) + data = array_data_from_json(file) if ext == FileFormats.JSONL else read_single_row_array_data_table(file) return data diff --git a/biodb/apps/uploader/loaddata.py b/biodb/apps/uploader/loaddata.py index 6b7a90c..97c8b40 100644 --- a/biodb/apps/uploader/loaddata.py +++ b/biodb/apps/uploader/loaddata.py @@ -14,17 +14,17 @@ class ExitTransaction(Exception): ... -def save_data_to_db(meta_data, spectral_data, center=None, joined_data=None, dry_run=False) -> dict: +def save_data_to_db(meta_data, array_data, center=None, joined_data=None, dry_run=False) -> dict: """ Ingest into the database large tables of observation & observable data (aka "meta" data) along with associated - spectral data. + array data. Note: Data can be passed in pre-joined, i.e., save_data_to_db(None, None, joined_data). If so, data can't be validated. Note: This func is called by UploadedFile.clean() which, therefore, can't also be called here. """ - from uploader.models import BioSample, Observable, Instrument, Patient, SpectralData, Observation, UploadedFile,\ + from uploader.models import BioSample, Observable, Instrument, Patient, ArrayData, Observation, UploadedFile,\ Visit, Center as UploaderCenter from user.models import Center as UserCenter @@ -39,15 +39,15 @@ def save_data_to_db(meta_data, spectral_data, center=None, joined_data=None, dry # Read in all data. meta_data = meta_data if isinstance(meta_data, pd.DataFrame) else \ uploader.io.read_meta_data(meta_data, index_column=index_column) - spec_data = spectral_data if isinstance(spectral_data, pd.DataFrame) else \ - uploader.io.read_spectral_data_table(spectral_data, index_column=index_column) + spec_data = array_data if isinstance(array_data, pd.DataFrame) else \ + uploader.io.read_array_data_table(array_data, index_column=index_column) UploadedFile.validate_lengths(meta_data, spec_data) joined_data = UploadedFile.join_with_validation(meta_data, spec_data) try: with transaction.atomic(using="bsr"): - spectral_data_files = [] + array_data_files = [] # Ingest into db. for index, row in joined_data.iterrows(): @@ -78,27 +78,27 @@ def save_data_to_db(meta_data, spectral_data, center=None, joined_data=None, dry biosample.save() visit.bio_sample.add(biosample, bulk=False) - # SpectralData + # ArrayData instrument = get_object_or_raise_validation(Instrument, pk=row.get("instrument")) # Create datafile - json_str = uploader.io.spectral_data_to_json(file=None, + json_str = uploader.io.array_data_to_json(file=None, data=None, patient_id=patient.patient_id, wavelength=row["wavelength"], intensity=row["intensity"]) - spectraldata = SpectralData(instrument=instrument, + arraydata = ArrayData(instrument=instrument, bio_sample=biosample, - **SpectralData.parse_fields_from_pandas_series(row)) - filename = f"{uploader.io.TEMP_FILENAME_PREFIX if dry_run else ''}{spectraldata.generate_filename()}" - spectraldata.data = ContentFile(json_str, name=filename) - spectraldata.full_clean() - spectraldata.save() - spectral_data_files.append(spectraldata.data) + **ArrayData.parse_fields_from_pandas_series(row)) + filename = f"{uploader.io.TEMP_FILENAME_PREFIX if dry_run else ''}{arraydata.generate_filename()}" + arraydata.data = ContentFile(json_str, name=filename) + arraydata.full_clean() + arraydata.save() + array_data_files.append(arraydata.data) - biosample.spectral_data.add(spectraldata, bulk=False) - instrument.spectral_data.add(spectraldata, bulk=False) + biosample.array_data.add(arraydata, bulk=False) + instrument.array_data.add(arraydata, bulk=False) # Observations for observable in Observable.objects.filter(Q(center=center) | Q(center=None)): @@ -121,16 +121,16 @@ def save_data_to_db(meta_data, spectral_data, center=None, joined_data=None, dry pass except Exception: # Something went wrong and the above transaction was aborted so delete uncommitted and now orphaned files. - while spectral_data_files: - file = spectral_data_files.pop() + while array_data_files: + file = array_data_files.pop() if not file.closed: file.close() - SpectralData.data.field.storage.delete(file.name) # Pop to avoid repetition in finally branch. + ArrayData.data.field.storage.delete(file.name) # Pop to avoid repetition in finally branch. raise finally: # Delete unwanted temporary files. - for file in spectral_data_files: + for file in array_data_files: if (filename := Path(file.name)).name.startswith(uploader.io.TEMP_FILENAME_PREFIX): if not file.closed: file.close() - SpectralData.data.field.storage.delete(filename) + ArrayData.data.field.storage.delete(filename) diff --git a/biodb/apps/uploader/management/commands/run_qc_annotators.py b/biodb/apps/uploader/management/commands/run_qc_annotators.py index 660292f..38ffcae 100644 --- a/biodb/apps/uploader/management/commands/run_qc_annotators.py +++ b/biodb/apps/uploader/management/commands/run_qc_annotators.py @@ -1,9 +1,9 @@ from django.core.management.base import BaseCommand, CommandError -from uploader.models import SpectralData, QCAnnotator +from uploader.models import ArrayData, QCAnnotator class Command(BaseCommand): - help = "Run all Quality Control annotators on the SpectralData database table." + help = "Run all Quality Control annotators on the ArrayData database table." def add_arguments(self, parser): parser.add_argument("--no_reruns", @@ -24,14 +24,14 @@ def handle(self, *args, **options): return try: - all_data = SpectralData.objects.all() + all_data = ArrayData.objects.all() except Exception: raise CommandError(f"An error occurred when trying to retrieve all entries from the" - f" {SpectralData.__name__} table.") + f" {ArrayData.__name__} table.") if not all_data: self.stdout.write( - self.style.WARNING("No SpectralData exists to annotate.") + self.style.WARNING("No ArrayData exists to annotate.") ) return @@ -39,7 +39,7 @@ def handle(self, *args, **options): n_annotators = all_annotators.count() n_data = all_data.count() self.stdout.write(f"There are {n_annotators} annotators and {n_data} entries in the" - f" '{SpectralData.__name__}' table to annotate." + f" '{ArrayData.__name__}' table to annotate." f"\nThat's {n_annotators * n_data} annotations in total.") for i, data in enumerate(all_data): diff --git a/biodb/apps/uploader/migrations/0001_initial.py b/biodb/apps/uploader/migrations/0001_initial.py index 7094afd..61cb5eb 100644 --- a/biodb/apps/uploader/migrations/0001_initial.py +++ b/biodb/apps/uploader/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.11 on 2024-03-13 00:13 +# Generated by Django 4.2.11 on 2024-05-30 01:49 import django.core.validators from django.db import migrations, models @@ -57,26 +57,15 @@ class Migration(migrations.Migration): bases=(uploader.base_models.SqlView, models.Model), ), migrations.CreateModel( - name='BioSample', + name='ArrayMeasurementType', fields=[ ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('created_at', models.DateTimeField(auto_now_add=True)), ('updated_at', models.DateTimeField(auto_now=True)), - ('sample_cid', models.CharField(blank=True, max_length=256, null=True, verbose_name='Sample CID')), - ('sample_study_id', models.CharField(blank=True, max_length=256, null=True, verbose_name='Sample Study ID')), - ('sample_study_name', models.CharField(blank=True, max_length=256, null=True, verbose_name='Sample Study Name')), - ('sample_processing', models.CharField(blank=True, max_length=128, null=True, verbose_name='Sample Processing Description')), - ('sample_extraction', models.CharField(blank=True, max_length=128, null=True, verbose_name='Sample Extraction Description')), - ('sample_extraction_tube', models.CharField(blank=True, max_length=128, null=True, verbose_name='Sample Extraction Tube Brand Name')), - ('centrifuge_time', models.IntegerField(blank=True, null=True, verbose_name='Extraction Tube Centrifuge Time (s)')), - ('centrifuge_rpm', models.IntegerField(blank=True, null=True, verbose_name='Extraction Tube Centrifuge RPM')), - ('freezing_temp', models.FloatField(blank=True, null=True, verbose_name='Freezing Temperature (C)')), - ('thawing_temp', models.FloatField(blank=True, null=True, verbose_name='Thawing Temperature (C)')), - ('thawing_time', models.IntegerField(blank=True, null=True, verbose_name='Thawing time (s)')), - ('freezing_time', models.IntegerField(blank=True, null=True, verbose_name='Freezing time (s)')), + ('name', models.CharField(max_length=128, verbose_name='Array Measurement')), ], options={ - 'db_table': 'bio_sample', + 'db_table': 'array_measurement_type', 'get_latest_by': 'updated_at', }, ), @@ -105,29 +94,6 @@ class Migration(migrations.Migration): 'unique_together': {('name', 'country')}, }, ), - migrations.CreateModel( - name='Instrument', - fields=[ - ('created_at', models.DateTimeField(auto_now_add=True)), - ('updated_at', models.DateTimeField(auto_now=True)), - ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False, unique=True, verbose_name='instrument id')), - ('cid', models.CharField(max_length=128, verbose_name='instrument cid')), - ('manufacturer', models.CharField(max_length=128, verbose_name='Instrument manufacturer')), - ('model', models.CharField(max_length=128, verbose_name='Instrument model')), - ('serial_number', models.CharField(max_length=128, verbose_name='Instrument SN#')), - ('spectrometer_manufacturer', models.CharField(max_length=128, verbose_name='Spectrometer manufacturer')), - ('spectrometer_model', models.CharField(max_length=128, verbose_name='Spectrometer model')), - ('spectrometer_serial_number', models.CharField(max_length=128, verbose_name='Spectrometer SN#')), - ('laser_manufacturer', models.CharField(max_length=128, verbose_name='Laser manufacturer')), - ('laser_model', models.CharField(max_length=128, verbose_name='Laser model')), - ('laser_serial_number', models.CharField(max_length=128, verbose_name='Laser SN#')), - ('center', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to='uploader.center')), - ], - options={ - 'db_table': 'instrument', - 'get_latest_by': 'updated_at', - }, - ), migrations.CreateModel( name='Observable', fields=[ @@ -154,7 +120,7 @@ class Migration(migrations.Migration): ('created_at', models.DateTimeField(auto_now_add=True)), ('updated_at', models.DateTimeField(auto_now=True)), ('patient_id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False, unique=True, verbose_name='Patient ID')), - ('patient_cid', models.CharField(blank=True, help_text='Patient ID prescribed by the associated center', max_length=128, null=True)), + ('patient_cid', models.UUIDField(blank=True, help_text='Patient ID prescribed by the associated center', null=True)), ('center', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='uploader.center')), ], options={ @@ -173,26 +139,13 @@ class Migration(migrations.Migration): ('fully_qualified_class_name', models.CharField(help_text="This must be the fully qualified Python name for an implementation of QCFilter, e.g.,'myProject.qc.myQCFilter'.", max_length=128, unique=True, validators=[uploader.models.validate_qc_annotator_import])), ('value_type', models.CharField(choices=[('BOOL', 'Bool'), ('STR', 'Str'), ('INT', 'Int'), ('FLOAT', 'Float')], default='BOOL', max_length=128)), ('description', models.CharField(blank=True, max_length=256, null=True)), - ('default', models.BooleanField(default=True, help_text='If True it will apply to all spectral data samples.')), + ('default', models.BooleanField(default=True, help_text='If True it will apply to all array data samples.')), ], options={ 'db_table': 'qc_annotator', 'get_latest_by': 'updated_at', }, ), - migrations.CreateModel( - name='SpectraMeasurementType', - fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('created_at', models.DateTimeField(auto_now_add=True)), - ('updated_at', models.DateTimeField(auto_now=True)), - ('name', models.CharField(max_length=128, verbose_name='Spectra Measurement')), - ], - options={ - 'db_table': 'spectra_measurement_type', - 'get_latest_by': 'updated_at', - }, - ), migrations.CreateModel( name='Visit', fields=[ @@ -215,7 +168,7 @@ class Migration(migrations.Migration): ('created_at', models.DateTimeField(auto_now_add=True)), ('updated_at', models.DateTimeField(auto_now=True)), ('meta_data_file', models.FileField(help_text='File containing rows of all patient, observation, and other meta data.', upload_to='raw_data/', validators=[django.core.validators.FileExtensionValidator(['csv', 'xlsx', 'jsonl'])])), - ('spectral_data_file', models.FileField(help_text='File containing rows of spectral intensities for the corresponding meta data file.', upload_to='raw_data/', validators=[django.core.validators.FileExtensionValidator(['csv', 'xlsx', 'jsonl'])])), + ('array_data_file', models.FileField(help_text='File containing rows of array data for the corresponding meta data file.', upload_to='raw_data/', validators=[django.core.validators.FileExtensionValidator(['csv', 'xlsx', 'jsonl'])])), ('center', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='uploader.center')), ], options={ @@ -225,62 +178,89 @@ class Migration(migrations.Migration): }, ), migrations.CreateModel( - name='SpectralData', + name='Observation', fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('created_at', models.DateTimeField(auto_now_add=True)), ('updated_at', models.DateTimeField(auto_now=True)), - ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False, unique=True)), - ('measurement_id', models.CharField(blank=True, max_length=128, null=True)), - ('atr_crystal', models.CharField(blank=True, max_length=128, null=True, verbose_name='ATR Crystal')), - ('n_coadditions', models.IntegerField(blank=True, null=True, verbose_name='Number of coadditions')), - ('acquisition_time', models.IntegerField(blank=True, null=True, verbose_name='Acquisition time [s]')), - ('resolution', models.IntegerField(blank=True, null=True, verbose_name='Resolution [cm-1]')), - ('power', models.FloatField(blank=True, max_length=128, null=True, verbose_name='Power incident to the sample [mW]')), - ('temperature', models.FloatField(blank=True, max_length=128, null=True, verbose_name='Temperature [C]')), - ('pressure', models.FloatField(blank=True, max_length=128, null=True, verbose_name='Pressure [bar]')), - ('humidity', models.FloatField(blank=True, max_length=128, null=True, verbose_name='Humidity [%]')), - ('date', models.DateTimeField(blank=True, null=True)), - ('sers_description', models.CharField(blank=True, max_length=128, null=True, verbose_name='SERS description')), - ('sers_particle_material', models.CharField(blank=True, max_length=128, null=True, verbose_name='SERS particle material')), - ('sers_particle_size', models.FloatField(blank=True, null=True, verbose_name='SERS particle size [μm]')), - ('sers_particle_concentration', models.FloatField(blank=True, null=True, verbose_name='SERS particle concentration')), - ('data', models.FileField(max_length=256, upload_to='spectral_data/', validators=[django.core.validators.FileExtensionValidator(['csv', 'xlsx', 'jsonl'])], verbose_name='Spectral data file')), - ('bio_sample', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='spectral_data', to='uploader.biosample')), - ('instrument', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='spectral_data', to='uploader.instrument')), - ('measurement_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='spectral_data', to='uploader.spectrameasurementtype', verbose_name='Measurement type')), + ('days_observed', models.IntegerField(blank=True, default=None, help_text='Supersedes Visit.days_observed', null=True, validators=[django.core.validators.MinValueValidator(0)], verbose_name='Days observed')), + ('observable_value', models.CharField(blank=True, default='', max_length=128, null=True)), + ('observable', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='observation', to='uploader.observable')), + ('visit', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='observation', to='uploader.visit')), ], options={ - 'verbose_name': 'Spectral Data', - 'verbose_name_plural': 'Spectral Data', - 'db_table': 'spectral_data', + 'db_table': 'observation', 'get_latest_by': 'updated_at', }, ), migrations.CreateModel( - name='Observation', + name='Instrument', fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('created_at', models.DateTimeField(auto_now_add=True)), ('updated_at', models.DateTimeField(auto_now=True)), - ('days_observed', models.IntegerField(blank=True, default=None, help_text='Supersedes Visit.days_observed', null=True, validators=[django.core.validators.MinValueValidator(0)], verbose_name='Days observed')), - ('observable_value', models.CharField(blank=True, default='', max_length=128, null=True)), - ('observable', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='observation', to='uploader.observable')), - ('visit', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='observation', to='uploader.visit')), + ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False, unique=True, verbose_name='instrument id')), + ('cid', models.CharField(max_length=128, verbose_name='instrument cid')), + ('manufacturer', models.CharField(max_length=128, verbose_name='Instrument manufacturer')), + ('model', models.CharField(max_length=128, verbose_name='Instrument model')), + ('serial_number', models.CharField(max_length=128, verbose_name='Instrument SN#')), + ('center', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to='uploader.center')), ], options={ - 'db_table': 'observation', + 'db_table': 'instrument', 'get_latest_by': 'updated_at', }, ), - migrations.AddField( - model_name='biosample', - name='sample_type', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='bio_sample', to='uploader.biosampletype', verbose_name='Sample Type'), + migrations.CreateModel( + name='BioSample', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('sample_cid', models.CharField(blank=True, max_length=256, null=True, verbose_name='Sample CID')), + ('sample_study_id', models.CharField(blank=True, max_length=256, null=True, verbose_name='Sample Study ID')), + ('sample_study_name', models.CharField(blank=True, max_length=256, null=True, verbose_name='Sample Study Name')), + ('sample_processing', models.CharField(blank=True, max_length=128, null=True, verbose_name='Sample Processing Description')), + ('sample_extraction', models.CharField(blank=True, max_length=128, null=True, verbose_name='Sample Extraction Description')), + ('sample_extraction_tube', models.CharField(blank=True, max_length=128, null=True, verbose_name='Sample Extraction Tube Brand Name')), + ('centrifuge_time', models.IntegerField(blank=True, null=True, verbose_name='Extraction Tube Centrifuge Time [s]')), + ('centrifuge_rpm', models.IntegerField(blank=True, null=True, verbose_name='Extraction Tube Centrifuge RPM')), + ('freezing_temp', models.FloatField(blank=True, null=True, verbose_name='Freezing Temperature [C]')), + ('thawing_temp', models.FloatField(blank=True, null=True, verbose_name='Thawing Temperature [C]')), + ('thawing_time', models.FloatField(blank=True, null=True, verbose_name='Thawing time [minutes]')), + ('freezing_time', models.FloatField(blank=True, null=True, verbose_name='Freezing time [days]')), + ('sample_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='bio_sample', to='uploader.biosampletype', verbose_name='Sample Type')), + ('visit', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='bio_sample', to='uploader.visit')), + ], + options={ + 'db_table': 'bio_sample', + 'get_latest_by': 'updated_at', + }, ), - migrations.AddField( - model_name='biosample', - name='visit', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='bio_sample', to='uploader.visit'), + migrations.CreateModel( + name='ArrayData', + fields=[ + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False, unique=True)), + ('measurement_id', models.CharField(blank=True, max_length=128, null=True)), + ('acquisition_time', models.IntegerField(blank=True, null=True, verbose_name='Acquisition time [s]')), + ('resolution', models.IntegerField(blank=True, null=True, verbose_name='Resolution [1/cm]')), + ('power', models.FloatField(blank=True, max_length=128, null=True, verbose_name='Power incident to the sample [mW]')), + ('temperature', models.FloatField(blank=True, max_length=128, null=True, verbose_name='Temperature [C]')), + ('pressure', models.FloatField(blank=True, max_length=128, null=True, verbose_name='Pressure [bar]')), + ('humidity', models.FloatField(blank=True, max_length=128, null=True, verbose_name='Humidity [%]')), + ('date', models.DateTimeField(blank=True, null=True)), + ('data', models.FileField(max_length=256, upload_to='array_data/', validators=[django.core.validators.FileExtensionValidator(['csv', 'xlsx', 'jsonl'])], verbose_name='Array data file')), + ('bio_sample', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='array_data', to='uploader.biosample')), + ('instrument', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='array_data', to='uploader.instrument')), + ('measurement_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='array_data', to='uploader.arraymeasurementtype', verbose_name='Measurement type')), + ], + options={ + 'verbose_name': 'Array Data', + 'verbose_name_plural': 'Array Data', + 'db_table': 'array_data', + 'get_latest_by': 'updated_at', + }, ), migrations.CreateModel( name='QCAnnotation', @@ -290,12 +270,12 @@ class Migration(migrations.Migration): ('updated_at', models.DateTimeField(auto_now=True)), ('value', models.CharField(blank=True, max_length=128, null=True)), ('annotator', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='qc_annotation', to='uploader.qcannotator')), - ('spectral_data', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='qc_annotation', to='uploader.spectraldata')), + ('array_data', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='qc_annotation', to='uploader.arraydata')), ], options={ 'db_table': 'qc_annotation', 'get_latest_by': 'updated_at', - 'unique_together': {('annotator', 'spectral_data')}, + 'unique_together': {('annotator', 'array_data')}, }, ), migrations.AddConstraint( diff --git a/biodb/apps/uploader/migrations/0002_alter_patient_patient_cid.py b/biodb/apps/uploader/migrations/0002_alter_patient_patient_cid.py deleted file mode 100644 index 117b689..0000000 --- a/biodb/apps/uploader/migrations/0002_alter_patient_patient_cid.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 4.2.11 on 2024-04-05 00:34 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('uploader', '0001_initial'), - ] - - operations = [ - migrations.AlterField( - model_name='patient', - name='patient_cid', - field=models.UUIDField(blank=True, help_text='Patient ID prescribed by the associated center', null=True), - ), - ] diff --git a/biodb/apps/uploader/migrations/0003_alter_biosample_freezing_time_and_more.py b/biodb/apps/uploader/migrations/0003_alter_biosample_freezing_time_and_more.py deleted file mode 100644 index 9dfb8e7..0000000 --- a/biodb/apps/uploader/migrations/0003_alter_biosample_freezing_time_and_more.py +++ /dev/null @@ -1,23 +0,0 @@ -# Generated by Django 4.2.11 on 2024-04-16 13:39 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('uploader', '0002_alter_patient_patient_cid'), - ] - - operations = [ - migrations.AlterField( - model_name='biosample', - name='freezing_time', - field=models.FloatField(blank=True, null=True, verbose_name='Freezing time (days)'), - ), - migrations.AlterField( - model_name='biosample', - name='thawing_time', - field=models.IntegerField(blank=True, null=True, verbose_name='Thawing time (minutes)'), - ), - ] diff --git a/biodb/apps/uploader/migrations/0004_alter_biosample_thawing_time.py b/biodb/apps/uploader/migrations/0004_alter_biosample_thawing_time.py deleted file mode 100644 index 64eeec5..0000000 --- a/biodb/apps/uploader/migrations/0004_alter_biosample_thawing_time.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 4.2.11 on 2024-04-18 14:19 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('uploader', '0003_alter_biosample_freezing_time_and_more'), - ] - - operations = [ - migrations.AlterField( - model_name='biosample', - name='thawing_time', - field=models.FloatField(blank=True, null=True, verbose_name='Thawing time (minutes)'), - ), - ] diff --git a/biodb/apps/uploader/migrations/0005_alter_biosample_centrifuge_time_and_more.py b/biodb/apps/uploader/migrations/0005_alter_biosample_centrifuge_time_and_more.py deleted file mode 100644 index 6c510f8..0000000 --- a/biodb/apps/uploader/migrations/0005_alter_biosample_centrifuge_time_and_more.py +++ /dev/null @@ -1,43 +0,0 @@ -# Generated by Django 4.2.11 on 2024-05-09 15:27 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('uploader', '0004_alter_biosample_thawing_time'), - ] - - operations = [ - migrations.AlterField( - model_name='biosample', - name='centrifuge_time', - field=models.IntegerField(blank=True, null=True, verbose_name='Extraction Tube Centrifuge Time [s]'), - ), - migrations.AlterField( - model_name='biosample', - name='freezing_temp', - field=models.FloatField(blank=True, null=True, verbose_name='Freezing Temperature [C]'), - ), - migrations.AlterField( - model_name='biosample', - name='freezing_time', - field=models.FloatField(blank=True, null=True, verbose_name='Freezing time [days]'), - ), - migrations.AlterField( - model_name='biosample', - name='thawing_temp', - field=models.FloatField(blank=True, null=True, verbose_name='Thawing Temperature [C]'), - ), - migrations.AlterField( - model_name='biosample', - name='thawing_time', - field=models.FloatField(blank=True, null=True, verbose_name='Thawing time [minutes]'), - ), - migrations.AlterField( - model_name='spectraldata', - name='resolution', - field=models.IntegerField(blank=True, null=True, verbose_name='Resolution [1/cm]'), - ), - ] diff --git a/biodb/apps/uploader/models.py b/biodb/apps/uploader/models.py index d91f75c..9fe372e 100644 --- a/biodb/apps/uploader/models.py +++ b/biodb/apps/uploader/models.py @@ -81,12 +81,12 @@ def center_deletion_handler(sender, **kwargs): class UploadedFile(DatedModel): - """ Model for ingesting bulk data uploads of both spectral and meta data files. + """ Model for ingesting bulk data uploads of both array data and meta data files. Attributes: meta_data_file (:obj:`django.models.FileField`): The uploaded file containing rows of patient meta-data, e.g., observations, biosample collection info, etc. - spectral_data_file (:obj:`django.models.FileField`): The uploaded file containing rows of patient spectral + array_data_file (:obj:`django.models.FileField`): The uploaded file containing rows of patient array data. center (:obj:`django.models.ForeignKey` of :obj:`user.models.Center`): The center that all new uploaded patients will be associated to. @@ -104,17 +104,17 @@ class Meta: validators=[FileExtensionValidator(uploader.io.FileFormats.choices())], help_text="File containing rows of all patient, observation, and other meta" " data.") - spectral_data_file = models.FileField(upload_to=UPLOAD_DIR, - validators=[FileExtensionValidator(uploader.io.FileFormats.choices())], - help_text="File containing rows of spectral intensities for the corresponding" - " meta data file.") + array_data_file = models.FileField(upload_to=UPLOAD_DIR, + validators=[FileExtensionValidator(uploader.io.FileFormats.choices())], + help_text="File containing rows of array data for the corresponding" + " meta data file.") center = models.ForeignKey(Center, null=False, blank=False, on_delete=models.PROTECT) @staticmethod def validate_lengths(meta_data, spec_data): """ Validate that files must be of equal length (same number of rows). """ if len(meta_data) != len(spec_data): - raise ValidationError(_("meta and spectral data must be of equal length (%(a)i!=%(b)i)."), + raise ValidationError(_("meta and array data must be of equal length (%(a)i!=%(b)i)."), params={"a": len(meta_data), "b": len(spec_data)}, code="invalid") @@ -124,14 +124,14 @@ def join_with_validation(meta_data, spec_data): if not meta_data.index.equals(spec_data.index): raise ValidationError(_("Patient index mismatch. indexes from %(a)s must exactly match all those from %(b)s"), params=dict(a=UploadedFile.meta_data_file.field.name, - b=UploadedFile.spectral_data_file.field.name), + b=UploadedFile.array_data_file.field.name), code="invalid") try: # The simplest way to do this is to utilize pandas.DataFrame.join(). return meta_data.join(spec_data, how="left", validate="1:1") # Might as well return the join. except pd.errors.MergeError as error: - raise ValidationError(_("meta and spectral data must have unique and identical patient identifiers")) from error + raise ValidationError(_("meta and array data must have unique and identical patient identifiers")) from error def _validate_and_save_data_to_db(self, dry_run=False): try: @@ -139,12 +139,12 @@ def _validate_and_save_data_to_db(self, dry_run=False): # Note: When accessing ``models.FileField`` Django returns ``models.FieldFile`` as a proxy. meta_data = uploader.io.read_meta_data(self.meta_data_file.file, index_column=settings.BULK_UPLOAD_INDEX_COLUMN_NAME) - spec_data = uploader.io.read_spectral_data_table(self.spectral_data_file.file, - index_column=settings.BULK_UPLOAD_INDEX_COLUMN_NAME) + array_data = uploader.io.read_array_data_table(self.array_data_file.file, + index_column=settings.BULK_UPLOAD_INDEX_COLUMN_NAME) # Validate. - UploadedFile.validate_lengths(meta_data, spec_data) + UploadedFile.validate_lengths(meta_data, array_data) # This uses a join so returns the joined data so that it doesn't go to waste if needed, which it is here. - joined_data = UploadedFile.join_with_validation(meta_data, spec_data) + joined_data = UploadedFile.join_with_validation(meta_data, array_data) # Ingest into DB. save_data_to_db(None, None, center=self.center, joined_data=joined_data, dry_run=dry_run) @@ -170,7 +170,7 @@ def delete(self, *args, delete_files=True, **kwargs): if count == 1: if delete_files: self.meta_data_file.storage.delete(self.meta_data_file.name) - self.spectral_data_file.storage.delete(self.spectral_data_file.name) + self.array_data_file.storage.delete(self.array_data_file.name) return count, deleted def adelete(self, *args, **kwargs): @@ -187,9 +187,9 @@ def get_orphan_files(cls): return storage, {} # Collect all media files referenced in the DB. meta_data_files = set(x.meta_data_file.name for x in cls.objects.all()) - spectral_data_files = set(x.spectral_data_file.name for x in cls.objects.all()) + array_data_files = set(x.array_data_file.name for x in cls.objects.all()) # Compute orphaned file list. - orphaned_files = fs_files - (meta_data_files | spectral_data_files) + orphaned_files = fs_files - (meta_data_files | array_data_files) return storage, orphaned_files @@ -247,7 +247,7 @@ def clean(self): class Visit(DatedModel): - """ Model a patient's visitation to collect health data and biological samples. + """ Model a patient's visitation to collect health data and biological data. Attributes: patient (:obj:`django.models.ForeignKey` of :obj:`Patient`): The patient that this visit belongs to. @@ -567,12 +567,9 @@ def __str__(self): class Instrument(DatedModel): - """ Model the instrument/device used to measure spectral data. + """ Model the instrument/device used to measure array data. - Instruments are often comprised of both a spectrometer and laser source for which both of these may have their - own model and SN, etc. - - Note: This is not the collection method of the bio sample but the device used to spectroscopically analyze the + Note: This is not the collection method of the bio sample but the device used to analyze the biosample. Attributes: @@ -581,12 +578,6 @@ class Instrument(DatedModel): manufacturer (:obj:`django.models.CharField`): Instrument manufacturer name. model (:obj:`django.models.CharField`): Instrument model name. serial_number (:obj:`django.models.CharField`): Instrument serial number (SN#). - spectrometer_manufacturer (:obj:`django.models.CharField`): Spectrometer manufacturer name. - spectrometer_model (:obj:`django.models.CharField`): Spectrometer model name. - spectrometer_serial_number (:obj:`django.models.CharField`): Spectrometer serial number (SN#). - laser_manufacturer (:obj:`django.models.CharField`): Laser manufacturer name. - laser_model (:obj:`django.models.CharField`): Laser model name. - laser_serial_number (:obj:`django.models.CharField`): Laser serial number (SN#). center (:obj:`django.models.ForeignKey` of :obj:`user.models.Center`, optional): The center that this instrument belongs to. If ``null`` this instrument belongs to all centers, e.g., one used at a central processing lab. @@ -603,16 +594,6 @@ class Meta: model = models.CharField(max_length=128, verbose_name="Instrument model") serial_number = models.CharField(max_length=128, verbose_name="Instrument SN#") - # Spectrometer. - spectrometer_manufacturer = models.CharField(max_length=128, verbose_name="Spectrometer manufacturer") - spectrometer_model = models.CharField(max_length=128, verbose_name="Spectrometer model") - spectrometer_serial_number = models.CharField(max_length=128, verbose_name="Spectrometer SN#") - - # Laser. - laser_manufacturer = models.CharField(max_length=128, verbose_name="Laser manufacturer") - laser_model = models.CharField(max_length=128, verbose_name="Laser model") - laser_serial_number = models.CharField(max_length=128, verbose_name="Laser SN#") - center = models.ForeignKey(Center, null=True, blank=True, on_delete=models.PROTECT) @classmethod @@ -732,69 +713,60 @@ def __str__(self): return f"{self.visit}_type:{self.sample_type}_pk{self.pk}" # NOTE: str(self.visit) contains patient ID. -class SpectraMeasurementType(DatedModel): - """ Spectral measurement type. +class ArrayMeasurementType(DatedModel): + """ Array measurement type. Attributes: - name (:obj:`django.models.CharField`): Name of type, e.g., atr-ftir. + name (:obj:`django.models.CharField`): Name of type. """ class Meta: - db_table = "spectra_measurement_type" + db_table = "array_measurement_type" get_latest_by = "updated_at" - name = models.CharField(max_length=128, verbose_name="Spectra Measurement") + name = models.CharField(max_length=128, verbose_name="Array Measurement") def __str__(self): return self.name -class SpectralData(DatedModel): - """ Model spectral data measured by spectrometer instrument. +class ArrayData(DatedModel): + """ Model array data measured by instrument from biosample data. Attributes: id (:obj:`django.models.UUIDField`): The database primary key for the entry, Autogenerated if not provided. - data (:obj:`django.models.FileField`): The uploaded file containing the spectral data. - instrument (:obj:`django.models.ForeignKey` of :obj:`Instrument`): The instrument used to spectroscopically - produce the data. + data (:obj:`django.models.FileField`): The uploaded file containing the array data. + instrument (:obj:`django.models.ForeignKey` of :obj:`Instrument`): The instrument used to measure and + produce the array data. bio_sample (:obj:`django.models.CharField` of :obj:`BioSample`): The biosample that was analyzed. - measurement_type (:obj:`django.models.ForeignKey` of :obj:`SpectralMeasurementType`): The measurement type - e.g., atr-ftir. + measurement_type (:obj:`django.models.ForeignKey` of :obj:`ArrayMeasurementType`): The measurement type. measurement_id (:obj:`django.models.CharField`, optional): Identifier string for data. - atr_crystal (:obj:`django.models.CharField`, optional): The ATR crystal used (if at all). - n_coadditions (:obj:`django.models.IntegerField`, optional): The number of co-additions (if relevant). acquisition_time (:obj:`django.models.IntegerField`, optional): The acquisition time [s]. resolution (:obj:`django.models.CharField`, IntegerField): The resolution [1/cm] power (:obj:`django.models.FloatField`, optional): Power incident to the sample [mW] temperature (:obj:`django.models.FloatField`, optional: Temperature [C]. pressure (:obj:`django.models.FloatField`, optional): Pressure [bar]. date (:obj:`django.models.DateTimeField`, optional): Humidity [%]. - sers_description (:obj:`django.models.CharField`, optional): SERS description. - sers_particle_material (:obj:`django.models.CharField`, optional): SERS particle material. - sers_particle_size (:obj:`django.models.CharField`, optional): SERS particle size [\u03BCm]. - sers_particle_concentration (:obj:`django.models.CharField`, optional): SERS particle concentration. """ class Meta: - db_table = "spectral_data" - verbose_name = "Spectral Data" + db_table = "array_data" + verbose_name = "Array Data" verbose_name_plural = verbose_name get_latest_by = "updated_at" - UPLOAD_DIR = "spectral_data/" # MEDIA_ROOT/spectral_data + UPLOAD_DIR = "array_data/" # MEDIA_ROOT/array_data id = models.UUIDField(unique=True, primary_key=True, default=uuid.uuid4) - instrument = models.ForeignKey(Instrument, on_delete=models.CASCADE, related_name="spectral_data") - bio_sample = models.ForeignKey(BioSample, on_delete=models.CASCADE, related_name="spectral_data") + instrument = models.ForeignKey(Instrument, on_delete=models.CASCADE, related_name="array_data") + bio_sample = models.ForeignKey(BioSample, on_delete=models.CASCADE, related_name="array_data") # Measurement info measurement_id = models.CharField(max_length=128, blank=True, null=True) - measurement_type = models.ForeignKey(SpectraMeasurementType, + measurement_type = models.ForeignKey(ArrayMeasurementType, on_delete=models.CASCADE, verbose_name="Measurement type", - related_name="spectral_data") - atr_crystal = models.CharField(max_length=128, blank=True, null=True, verbose_name="ATR Crystal") - n_coadditions = models.IntegerField(blank=True, null=True, verbose_name="Number of coadditions") + related_name="array_data") acquisition_time = models.IntegerField(blank=True, null=True, verbose_name="Acquisition time [s]") resolution = models.IntegerField(blank=True, null=True, verbose_name="Resolution [1/cm]") power = models.FloatField(max_length=128, blank=True, null=True, verbose_name="Power incident to the sample [mW]") @@ -803,39 +775,24 @@ class Meta: humidity = models.FloatField(max_length=128, blank=True, null=True, verbose_name="Humidity [%]") date = models.DateTimeField(blank=True, null=True) - # SERS info. - sers_description = models.CharField(max_length=128, blank=True, null=True, verbose_name="SERS description") - sers_particle_material = models.CharField(max_length=128, - blank=True, - null=True, - verbose_name="SERS particle material") - sers_particle_size = models.FloatField(blank=True, null=True, verbose_name="SERS particle size [\u03BCm]") - sers_particle_concentration = models.FloatField(blank=True, null=True, verbose_name="SERS particle concentration") - data = models.FileField(upload_to=UPLOAD_DIR, validators=[FileExtensionValidator(UploadedFile.FileFormats.choices())], max_length=256, - verbose_name="Spectral data file") + verbose_name="Array data file") @classmethod def parse_fields_from_pandas_series(cls, series): """ Parse the pandas series for field values returning a dict. """ measurement_type = lower(get_field_value(series, cls, "measurement_type")) - measurement_type = get_object_or_raise_validation(SpectraMeasurementType, name=measurement_type) + measurement_type = get_object_or_raise_validation(ArrayMeasurementType, name=measurement_type) return dict(measurement_type=measurement_type, acquisition_time=get_field_value(series, cls, "acquisition_time"), - n_coadditions=get_field_value(series, cls, "n_coadditions"), resolution=get_field_value(series, cls, "resolution"), - atr_crystal=get_field_value(series, cls, "atr_crystal"), power=get_field_value(series, cls, "power"), temperature=get_field_value(series, cls, "temperature"), pressure=get_field_value(series, cls, "pressure"), humidity=get_field_value(series, cls, "humidity"), - date=get_field_value(series, cls, "date"), - sers_description=get_field_value(series, cls, "sers_description"), - sers_particle_material=get_field_value(series, cls, "sers_particle_material"), - sers_particle_size=get_field_value(series, cls, "sers_particle_size"), - sers_particle_concentration=get_field_value(series, cls, "sers_particle_concentration")) + date=get_field_value(series, cls, "date")) @property def center(self): @@ -859,9 +816,9 @@ def get_unrun_annotators(self, existing_annotators=None): return list(set(all_default_annotators) - set(existing_annotators)) - def get_spectral_data(self): - """ Return spectral data as instance of uploader.io.SpectralData. """ - return uploader.io.spectral_data_from_json(self.data) + def get_array_data(self): + """ Return array data as instance of uploader.io.ArrayData. """ + return uploader.io.array_data_from_json(self.data) def generate_filename(self): return Path(f"{self.bio_sample.visit.patient.patient_id}_{self.bio_sample.pk}_{self.id}")\ @@ -870,7 +827,7 @@ def generate_filename(self): def clean_data_file(self): """ Read in data from uploaded file and store as json. - The schema is equivalent to `json.dumps(dataclasses.asdict(uploader.io.SpectralData))``. + The schema is equivalent to `json.dumps(dataclasses.asdict(uploader.io.ArrayData))``. """ # Note: self.data is a FieldFile and is never None so check is "empty" instead, i.e., self.data.name is None. @@ -878,7 +835,7 @@ def clean_data_file(self): return try: - data = uploader.io.read_spectral_data(self.data) + data = uploader.io.read_array_data(self.data) except uploader.io.DataSchemaError as error: raise ValidationError(_("%(msg)s"), params={"msg": error}) @@ -887,7 +844,7 @@ def clean_data_file(self): filename = self.data.name else: filename = self.generate_filename() - json_str = uploader.io.spectral_data_to_json(file=None, data=data) + json_str = uploader.io.array_data_to_json(file=None, data=data) return ContentFile(json_str, name=filename) @@ -899,7 +856,7 @@ def clean(self): #@transaction.atomic(using="bsr") # Really? Not sure if this even can be if run in background... def annotate(self, annotator=None, force=False) -> list: - """ Run the quality control annotation on the spectral data. """ + """ Run the quality control annotation on the array data. """ # TODO: This needs to return early and run in the background. existing_annotators = self.get_annotators() @@ -911,7 +868,7 @@ def annotate(self, annotator=None, force=False) -> list: return annotation = self.qc_annotation.get(annotator=annotator) else: - annotation = QCAnnotation(annotator=annotator, spectral_data=self) + annotation = QCAnnotation(annotator=annotator, arary_data=self) return [annotation.run()] annotations = [] @@ -924,7 +881,7 @@ def annotate(self, annotator=None, force=False) -> list: # Create new annotations. for annotator in new_annotators: - annotation = QCAnnotation(annotator=annotator, spectral_data=self) + annotation = QCAnnotation(annotator=annotator, array_data=self) annotations.append(annotation.run()) return annotations if annotations else None # Don't ret empty list. @@ -1058,7 +1015,7 @@ class Meta: @classmethod def sql(cls): - # WARNING: The data exporters and charts rely on the spectral data column := "data", so we special case this + # WARNING: The data exporters and charts rely on the array data column := "data", so we special case this # instead of using ``_create_field_str_list``. The charts also rely on the field "patient_id" being present, so # that too we special case. Besides, patient.center shouldn't be included and that's the only other field. sql = f""" @@ -1071,10 +1028,10 @@ def sql(cls): "sample_cid", "sample_study_name"])}, {cls._create_field_str_list("i", Instrument, extra_excluded_field_names=["id", "cid"])}, - {cls._create_field_str_list("smt", SpectraMeasurementType, extra_excluded_field_names=["id"])}, + {cls._create_field_str_list("smt", ArrayMeasurementType, extra_excluded_field_names=["id"])}, {cls._create_field_str_list("sd", - SpectralData, - extra_excluded_field_names=[SpectralData.data.field.name, + ArrayData, + extra_excluded_field_names=[ArrayData.data.field.name, "id", "measurement_id", "date"])}, @@ -1084,7 +1041,7 @@ def sql(cls): join visit v on p.patient_id=v.patient_id join bio_sample bs on bs.visit_id=v.id join bio_sample_type bst on bst.id=bs.sample_type_id - join spectral_data sd on sd.bio_sample_id=bs.id + join array_data sd on sd.bio_sample_id=bs.id join spectra_measurement_type smt on smt.id=sd.measurement_type_id join instrument i on i.id=sd.instrument_id left outer join v_visit_observations vs on vs.visit_id=v.id @@ -1118,7 +1075,7 @@ class QCAnnotator(DatedModel): value_type (:obj:`django.models.CharField`): Value type selected from: BOOL, STR, INT, FLOAT. description (:obj:`django.models.CharField`, optional): A verbose description of the annotator's semantics. default (:obj:`django.models.BooleanField`, optional): Specifies whether this annotator is considered a - "default" or "global" annotator that will can be automatically applied to all spectral data entries. + "default" or "global" annotator that will can be automatically applied to all array data entries. """ class Meta: @@ -1141,7 +1098,7 @@ class Meta: default = models.BooleanField(default=True, blank=False, null=False, - help_text="If True it will apply to all spectral data samples.") + help_text="If True it will apply to all array data samples.") def __str__(self): return f"{self.name}: {self.fully_qualified_class_name}" @@ -1158,26 +1115,26 @@ def save(self, *args, **kwargs): super().save(*args, **kwargs) if settings.RUN_DEFAULT_ANNOTATORS_WHEN_SAVED and self.default: - # Run annotator on all spectral data samples. - for data in SpectralData.objects.all(): + # Run annotator on all array data samples. + for data in ArrayData.objects.all(): # Since this annotator could have been altered (from django) rather than being new, annotations # of this annotator may already exist, thus we need to force them to be re-run. data.annotate(annotator=self, force=True) class QCAnnotation(DatedModel): - """ A Quality Control annotation of a spectral data entry. + """ A Quality Control annotation of a array data entry. Attributes: value (:obj:`django.models.CharField`): The actual annotation value/data/result. annotator (:obj:`django.models.ForeignKey` of :obj:`QCAnnotator`): The quality control annotator used to - produce annotate the spectral data entry. - spectral_data (:obj:`django.models.ForeignKey` of :obj:`SpectralData`): The annotated spectral data entry. + produce annotate the array data entry. + array_data (:obj:`django.models.ForeignKey` of :obj:`ArrayData`): The annotated array data entry. """ class Meta: db_table = "qc_annotation" - unique_together = [["annotator", "spectral_data"]] + unique_together = [["annotator", "array_data"]] get_latest_by = "updated_at" value = models.CharField(blank=True, null=True, max_length=128) @@ -1187,11 +1144,11 @@ class Meta: null=False, on_delete=models.CASCADE, related_name="qc_annotation") - spectral_data = models.ForeignKey(SpectralData, on_delete=models.CASCADE, related_name="qc_annotation") + array_data = models.ForeignKey(ArrayData, on_delete=models.CASCADE, related_name="qc_annotation") @property def center(self): - return self.spectral_data.bio_samaple.visit.patient.center + return self.array_data.bio_samaple.visit.patient.center def __str__(self): return f"{self.annotator.name}: {self.value}" @@ -1201,7 +1158,7 @@ def get_value(self): return self.annotator.cast(self.value) def run(self, save=True): - value = self.annotator.run(self.spectral_data) + value = self.annotator.run(self.array_data) self.value = value if save: diff --git a/biodb/apps/uploader/tests/conftest.py b/biodb/apps/uploader/tests/conftest.py index 1411a64..998e8b8 100644 --- a/biodb/apps/uploader/tests/conftest.py +++ b/biodb/apps/uploader/tests/conftest.py @@ -14,7 +14,7 @@ from biodb.util import find_package_location -from uploader.models import SpectralData, UploadedFile, Center +from uploader.models import ArrayData, UploadedFile, Center from user.models import Center as UserCenter DATA_PATH = Path(__file__).parent / "data" @@ -53,7 +53,7 @@ def rm_all_media_dirs(): from catalog.models import Dataset # Tidy up any created files. rm_dir(Path(UploadedFile.UPLOAD_DIR)) - rm_dir(Path(SpectralData.UPLOAD_DIR)) + rm_dir(Path(ArrayData.UPLOAD_DIR)) rm_dir(Path(Dataset.UPLOAD_DIR)) @@ -69,7 +69,7 @@ class Meta: model = Query title = Sequence(lambda n: f'My simple query {n}') - sql = "select * from spectral_data" + sql = "select * from array_data" description = "Stuff" connection = settings.EXPLORER_DEFAULT_CONNECTION created_by_user = SubFactory(UserFactory) @@ -149,20 +149,20 @@ def qcannotators(db, django_db_blocker): @pytest.fixture(scope="function") def mock_data(db, django_db_blocker, centers): # NOTE: Since this loads directly to the DB without any validation and thus call to loaddata(), no data files are - # present. If you need actual spectral data, use ``mock_data_from_files`` below instead. + # present. If you need actual array data, use ``mock_data_from_files`` below instead. with django_db_blocker.unblock(): call_command('loaddata', "--database=bsr", 'test_data.json') def bulk_upload(): meta_data_path = (DATA_PATH / "meta_data").with_suffix(UploadedFile.FileFormats.XLSX) - spectral_file_path = (DATA_PATH / "spectral_data").with_suffix(UploadedFile.FileFormats.XLSX) + array_file_path = (DATA_PATH / "array_data").with_suffix(UploadedFile.FileFormats.XLSX) with meta_data_path.open(mode="rb") as meta_data: - with spectral_file_path.open(mode="rb") as spectral_data: + with array_file_path.open(mode="rb") as array_data: data_upload = UploadedFile(meta_data_file=django.core.files.File(meta_data, name=meta_data_path.name), - spectral_data_file=django.core.files.File(spectral_data, - name=spectral_file_path.name), + array_data_file=django.core.files.File(array_data, + name=array_file_path.name), center=Center.objects.get(name="jhu")) data_upload.clean() data_upload.save() diff --git a/biodb/apps/uploader/tests/test_charts.py b/biodb/apps/uploader/tests/test_charts.py index b1bc09b..6336726 100644 --- a/biodb/apps/uploader/tests/test_charts.py +++ b/biodb/apps/uploader/tests/test_charts.py @@ -7,7 +7,7 @@ @pytest.fixture() def query(request): sql_marker = request.node.get_closest_marker("sql") - sql = sql_marker.args[0] if sql_marker and sql_marker.args[0] else "select * from spectral_data" + sql = sql_marker.args[0] if sql_marker and sql_marker.args[0] else "select * from array_data" q = SimpleQueryFactory(sql=sql) results = q.execute_query_only() diff --git a/biodb/apps/uploader/tests/test_exporters.py b/biodb/apps/uploader/tests/test_exporters.py index 11b498c..5fc88cf 100644 --- a/biodb/apps/uploader/tests/test_exporters.py +++ b/biodb/apps/uploader/tests/test_exporters.py @@ -7,7 +7,7 @@ from uploader.exporters import CSVExporter import uploader.io -from uploader.models import SpectralData +from uploader.models import ArrayData from uploader.tests.conftest import SimpleQueryFactory @@ -25,7 +25,7 @@ def csv_export(request, monkeypatch, mock_data_from_files): monkeypatch.setattr(settings, "EXPLORER_DATA_EXPORTERS_ALLOW_DATA_FILE_ALIAS", allow_aliases.args[0]) sql_marker = request.node.get_closest_marker("sql") - sql = sql_marker.args[0] if sql_marker and sql_marker.args[0] else "select * from spectral_data" + sql = sql_marker.args[0] if sql_marker and sql_marker.args[0] else "select * from array_data" q = SimpleQueryFactory(sql=sql) exporter = CSVExporter(query=q) @@ -47,7 +47,7 @@ def test_without_data_files(self, csv_export): @pytest.mark.include_data_files(True) @pytest.mark.parametrize(tuple(), - [pytest.param(marks=pytest.mark.sql("select data, data from spectral_data")), + [pytest.param(marks=pytest.mark.sql("select data, data from array_data")), pytest.param(marks=pytest.mark.sql(None))]) def test_no_duplicate_data_files(self, csv_export): assert zipfile.is_zipfile(csv_export) @@ -58,35 +58,35 @@ def test_no_duplicate_data_files(self, csv_export): def test_with_data_files_check_content(self, csv_export): z = zipfile.ZipFile(csv_export) namelist = z.namelist() - spectral_data_dir = Path(SpectralData.data.field.upload_to) + array_data_dir = Path(ArrayData.data.field.upload_to) - spectral_data_files = [] + array_data_files = [] query_data_file = [] for filename in namelist: - if Path(filename).parent == spectral_data_dir: - spectral_data_files.append(filename) + if Path(filename).parent == array_data_dir: + array_data_files.append(filename) else: query_data_file.append(filename) - assert len(spectral_data_files) == 10 + assert len(array_data_files) == 10 assert len(query_data_file) == 1 query_data_file = query_data_file[0] with z.open(query_data_file) as f: data = uploader.io._read_raw_data(f, ext=Path(query_data_file).suffix) assert len(data) == 10 - assert set(data[SpectralData.data.field.name]) == set(spectral_data_files) + assert set(data[ArrayData.data.field.name]) == set(array_data_files) - for filename in spectral_data_files: - data = uploader.io.read_spectral_data(filename) - assert isinstance(data, uploader.io.SpectralData) + for filename in array_data_files: + data = uploader.io.read_array_data(filename) + assert isinstance(data, uploader.io.ArrayData) assert {x.name for x in dataclasses.fields(data)} == {"patient_id", "wavelength", "intensity"} - assert data.patient_id == SpectralData.objects.get(data=filename).bio_sample.visit.patient.patient_id + assert data.patient_id == ArrayData.objects.get(data=filename).bio_sample.visit.patient.patient_id assert len(data.wavelength) == 1798 assert len(data.intensity) == 1798 @pytest.mark.include_data_files(True) - @pytest.mark.sql("select * from patient") # uploader_patient contains no spectral data. + @pytest.mark.sql("select * from patient") # uploader_patient contains no array data. def test_no_data(self, csv_export): assert isinstance(csv_export, str) diff --git a/biodb/apps/uploader/tests/test_io.py b/biodb/apps/uploader/tests/test_io.py index 5f11bca..b18ad6a 100644 --- a/biodb/apps/uploader/tests/test_io.py +++ b/biodb/apps/uploader/tests/test_io.py @@ -20,74 +20,74 @@ def json_data(): return json.load(fp) -class TestReadSingleRowSpectralDataTable: +class TestReadSingleRowArrayDataTable: @pytest.mark.parametrize("ext", uploader.io.FileFormats.list()) def test_read(self, json_data, ext): filename = SPECTRAL_FILE_PATH.with_suffix(ext) - data = uploader.io.read_single_row_spectral_data_table(filename) + data = uploader.io.read_single_row_array_data_table(filename) assert str(data.patient_id) == json_data["patient_id"] @pytest.mark.parametrize("ext", uploader.io.FileFormats.list()) def test_multiple_row_exception(self, ext): with pytest.raises(ValueError, match="The file read should contain only a single row"): - uploader.io.read_single_row_spectral_data_table((DATA_PATH/"spectral_data").with_suffix(ext), + uploader.io.read_single_row_array_data_table((DATA_PATH/"array_data").with_suffix(ext), index_column=settings.BULK_UPLOAD_INDEX_COLUMN_NAME) -class TestSpectralDataFromJson: +class TestArrayDataFromJson: def test_read(self, json_data): filename = SPECTRAL_FILE_PATH.with_suffix(uploader.io.FileFormats.JSONL) - assert uploader.io.spectral_data_from_json(filename) == uploader.io.SpectralData(**json_data) + assert uploader.io.array_data_from_json(filename) == uploader.io.ArrayData(**json_data) - def test_spectral_data_from_json_key_validation(self): + def test_array_data_from_json_key_validation(self): fake_data = ContentFile(json.dumps({"blah": "huh?", "wavelength": [], "something else": 1.0}), name=Path("fake_json").with_suffix(uploader.io.FileFormats.JSONL)) with pytest.raises(uploader.io.DataSchemaError, match="Schema error:"): - uploader.io.spectral_data_from_json(fake_data) + uploader.io.array_data_from_json(fake_data) def test_exceptions(self): with pytest.raises(ValueError, match="Incorrect file format"): - uploader.io.spectral_data_from_json("filename_without_an_extension") + uploader.io.array_data_from_json("filename_without_an_extension") -class TestSpectralDataToJson: +class TestArrayDataToJson: def test_data_as_dict(self, json_data): - json_str = uploader.io.spectral_data_to_json(None, data=json_data) + json_str = uploader.io.array_data_to_json(None, data=json_data) assert isinstance(json_str, str) assert json.loads(json_str) == json_data def test_data_as_dataclass(self, json_data): - json_str = uploader.io.spectral_data_to_json(None, data=uploader.io.SpectralData(**json_data)) + json_str = uploader.io.array_data_to_json(None, data=uploader.io.ArrayData(**json_data)) assert isinstance(json_str, str) assert json.loads(json_str) == json_data def test_data_as_kwargs(self, json_data): - json_str = uploader.io.spectral_data_to_json(None, data=None, **json_data) + json_str = uploader.io.array_data_to_json(None, data=None, **json_data) assert isinstance(json_str, str) assert json.loads(json_str) == json_data def test_data_as_filename(self, json_data): filename = Path("myjson").with_suffix(uploader.io.FileFormats.JSONL) # Write data. - filename = uploader.io.spectral_data_to_json(filename, json_data) + filename = uploader.io.array_data_to_json(filename, json_data) # Read data. - data = uploader.io.spectral_data_from_json(filename) - assert uploader.io.SpectralData(**json_data) == data + data = uploader.io.array_data_from_json(filename) + assert uploader.io.ArrayData(**json_data) == data def test_data_as_fp(self, json_data): filename = Path("myjson").with_suffix(uploader.io.FileFormats.JSONL) # Write data. with storages["default"].open(filename, mode='w') as fp: - uploader.io.spectral_data_to_json(fp, json_data) + uploader.io.array_data_to_json(fp, json_data) # Read data. - data = uploader.io.spectral_data_from_json(filename) - assert uploader.io.SpectralData(**json_data) == data + data = uploader.io.array_data_from_json(filename) + assert uploader.io.ArrayData(**json_data) == data class TestReadRawData: @pytest.mark.parametrize("ext", uploader.io.FileFormats.list()) def test_read(self, ext): - data = uploader.io._read_raw_data((DATA_PATH / "spectral_data").with_suffix(ext)) + data = uploader.io._read_raw_data((DATA_PATH / "array_data").with_suffix(ext)) assert len(data) == 10 def test_ext_exception(self): diff --git a/biodb/apps/uploader/tests/test_management_commands.py b/biodb/apps/uploader/tests/test_management_commands.py index 9ab2510..25a3277 100644 --- a/biodb/apps/uploader/tests/test_management_commands.py +++ b/biodb/apps/uploader/tests/test_management_commands.py @@ -8,7 +8,7 @@ from django.db.models import Q from django.db.utils import OperationalError -from uploader.models import Center, Observable, UploadedFile, SpectralData +from uploader.models import Center, Observable, UploadedFile, ArrayData from uploader.sql import execute_sql User = get_user_model() @@ -69,11 +69,11 @@ def get_file_count(path): # Check objects exist. assert UploadedFile.objects.count() == 1 - assert SpectralData.objects.count() == 10 + assert ArrayData.objects.count() == 10 # Check files exist. assert get_file_count(UploadedFile.UPLOAD_DIR) == 2 - assert get_file_count(SpectralData.UPLOAD_DIR) == 10 + assert get_file_count(ArrayData.UPLOAD_DIR) == 10 # Delete objects and files (if delete). def delete_objs(model): @@ -81,7 +81,7 @@ def delete_objs(model): obj.delete(delete_files=delete) assert model.objects.count() == 0 delete_objs(UploadedFile) - delete_objs(SpectralData) + delete_objs(ArrayData) # Run prune_files command. out = StringIO() @@ -89,7 +89,7 @@ def delete_objs(model): # Check files have been deleted. assert get_file_count(UploadedFile.UPLOAD_DIR) == expected[0] - assert get_file_count(SpectralData.UPLOAD_DIR) == expected[1] + assert get_file_count(ArrayData.UPLOAD_DIR) == expected[1] out.seek(0) assert len(out.readlines()) == 1 if delete else 13 diff --git a/biodb/apps/uploader/tests/test_models.py b/biodb/apps/uploader/tests/test_models.py index 8304509..1b84433 100644 --- a/biodb/apps/uploader/tests/test_models.py +++ b/biodb/apps/uploader/tests/test_models.py @@ -11,8 +11,8 @@ import pytest import uploader.io -from uploader.models import BioSample, BioSampleType, Observable, Instrument, Patient, SpectralData, Observation,\ - Visit, UploadedFile, get_center, Center, SpectraMeasurementType +from uploader.models import BioSample, BioSampleType, Observable, Instrument, Patient, ArrayData, Observation,\ + Visit, UploadedFile, get_center, Center, ArrayMeasurementType from uploader.loaddata import save_data_to_db from user.models import Center as UserCenter from uploader.models import Center as UploaderCenter @@ -333,32 +333,32 @@ class TestBioSample: @pytest.mark.django_db(databases=["default", "bsr"]) -class TestSpectralData: +class TestArrayData: def test_files_added(self, mock_data_from_files): n_patients = 10 - assert SpectralData.objects.count() == n_patients - for obj in SpectralData.objects.all(): + assert ArrayData.objects.count() == n_patients + for obj in ArrayData.objects.all(): assert Path(obj.data.name).exists() def test_no_file_validation(self, db): """ Test that a validation error is raised rather than any other python exception which would indicate a bug. """ - data = SpectralData() + data = ArrayData() with pytest.raises(ValidationError): data.full_clean() def test_temp_files_deleted(self, mock_data_from_files): n_patients = 10 - assert SpectralData.objects.count() == n_patients - filename = Path(SpectralData.objects.all()[0].data.name) + assert ArrayData.objects.count() == n_patients + filename = Path(ArrayData.objects.all()[0].data.name) assert filename.parent.exists() assert filename.parent.is_dir() assert not list(filename.parent.glob(f"{uploader.io.TEMP_FILENAME_PREFIX}*")) def test_no_duplicate_data_files(self, mock_data_from_files): n_patients = 10 - assert SpectralData.objects.count() == n_patients - filename = Path(SpectralData.objects.all()[0].data.name) + assert ArrayData.objects.count() == n_patients + filename = Path(ArrayData.objects.all()[0].data.name) assert filename.parent.exists() assert filename.parent.is_dir() assert len(list(filename.parent.glob('*'))) == n_patients @@ -371,31 +371,31 @@ def test_all_files_deleted_upon_transaction_failure(self): @pytest.mark.parametrize("ext", uploader.io.FileFormats.list()) def test_clean(self, centers, instruments, ext, bio_sample_types, spectra_measurement_types): data_file = (DATA_PATH/"sample").with_suffix(ext) - data = uploader.io.read_spectral_data(data_file) + data = uploader.io.read_array_data(data_file) patient = Patient.objects.create(patient_id=data.patient_id, center=Center.objects.get(name="jhu")) visit = patient.visit.create() bio_sample = visit.bio_sample.create(sample_type=BioSampleType.objects.get(name="pharyngeal swab")) - spectral_data = SpectralData(instrument=Instrument.objects.get(pk="4205d8ac-90c1-4529-90b2-6751f665c403"), + array_data = ArrayData(instrument=Instrument.objects.get(pk="4205d8ac-90c1-4529-90b2-6751f665c403"), bio_sample=bio_sample, data=ContentFile(data_file.read_bytes(), name=data_file), - measurement_type=SpectraMeasurementType.objects.get(name="atr-ftir")) - spectral_data.full_clean() - spectral_data.save() + measurement_type=ArrayMeasurementType.objects.get(name="atr-ftir")) + array_data.full_clean() + array_data.save() - assert Path(spectral_data.data.name).suffix == uploader.io.FileFormats.JSONL - cleaned_data = spectral_data.get_spectral_data() + assert Path(array_data.data.name).suffix == uploader.io.FileFormats.JSONL + cleaned_data = array_data.get_array_data() assert cleaned_data == data def test_deletion(self, mock_data_from_files): - spectral_data = SpectralData.objects.all() - for item in spectral_data: - assert SpectralData.data.field.storage.exists(item.data.name) + array_data = ArrayData.objects.all() + for item in array_data: + assert ArrayData.data.field.storage.exists(item.data.name) item.delete() - assert not SpectralData.data.field.storage.exists(item.data.name) - assert not SpectralData.objects.count() + assert not ArrayData.data.field.storage.exists(item.data.name) + assert not ArrayData.objects.count() @pytest.mark.django_db(databases=["default", "bsr"]) @@ -410,12 +410,12 @@ def test_upload_without_error(self, bio_sample_types, spectra_measurement_types): meta_data_path = (DATA_PATH/"meta_data").with_suffix(file_ext) - spectral_file_path = (DATA_PATH / "spectral_data").with_suffix(file_ext) - with meta_data_path.open(mode="rb") as meta_data, spectral_file_path.open(mode="rb") as spectral_data: + array_file_path = (DATA_PATH / "array_data").with_suffix(file_ext) + with meta_data_path.open(mode="rb") as meta_data, array_file_path.open(mode="rb") as array_data: data_upload = UploadedFile(meta_data_file=django.core.files.File(meta_data, name=meta_data_path.name), - spectral_data_file=django.core.files.File(spectral_data, - name=spectral_file_path.name), + array_data_file=django.core.files.File(array_data, + name=array_file_path.name), center=center) data_upload.clean() data_upload.save() @@ -426,7 +426,7 @@ def test_mock_data_from_files_fixture(self, mock_data_from_files): assert Patient.objects.count() == n_patients assert Visit.objects.count() == n_patients assert BioSample.objects.count() == n_patients - assert SpectralData.objects.count() == n_patients + assert ArrayData.objects.count() == n_patients def test_center(self, mock_data_from_files): n_patients = Patient.objects.count() @@ -440,7 +440,7 @@ def test_mock_data_fixture(self, mock_data): assert Patient.objects.count() == n_patients assert Visit.objects.count() == n_patients assert BioSample.objects.count() == n_patients - assert SpectralData.objects.count() == n_patients + assert ArrayData.objects.count() == n_patients def test_number_observations(self, db, @@ -453,7 +453,7 @@ def test_number_observations(self, assert Patient.objects.count() == 0 # Assert empty. save_data_to_db(DATA_PATH / "meta_data.csv", - DATA_PATH / "spectral_data.csv", + DATA_PATH / "array_data.csv", center=center) n_patients = Patient.objects.count() @@ -486,13 +486,13 @@ def test_patient_ids(self, mock_data_from_files, file_ext): def test_index_match_validation(self, db, observables, instruments, file_ext, tmp_path): meta_data_path = (DATA_PATH / "meta_data").with_suffix(file_ext) - biodb.util.mock_bulk_spectral_data(path=tmp_path) - spectral_file_path = tmp_path / "spectral_data.csv" - with meta_data_path.open(mode="rb") as meta_data, spectral_file_path.open(mode="rb") as spectral_data: + biodb.util.mock_bulk_array_data(path=tmp_path) + array_file_path = tmp_path / "array_data.csv" + with meta_data_path.open(mode="rb") as meta_data, array_file_path.open(mode="rb") as array_data: data_upload = UploadedFile(meta_data_file=django.core.files.File(meta_data, name=meta_data_path.name), - spectral_data_file=django.core.files.File(spectral_data, - name=spectral_file_path.name)) + array_data_file=django.core.files.File(array_data, + name=array_file_path.name)) with pytest.raises(ValidationError, match="Patient index mismatch."): data_upload.clean() @@ -552,11 +552,11 @@ def test_upload_on_cid(self, django_db_blocker, observables, instruments, mock_d def test_deletion(self, mock_data_from_files): bulk_upload = UploadedFile.objects.all()[0] assert os.path.exists(bulk_upload.meta_data_file.name) - assert os.path.exists(bulk_upload.spectral_data_file.name) + assert os.path.exists(bulk_upload.array_data_file.name) bulk_upload.delete() assert not UploadedFile.objects.count() assert not os.path.exists(bulk_upload.meta_data_file.name) - assert not os.path.exists(bulk_upload.spectral_data_file.name) + assert not os.path.exists(bulk_upload.array_data_file.name) def test_bad_ext(self, db, @@ -567,19 +567,19 @@ def test_bad_ext(self, bio_sample_types, spectra_measurement_types): meta_data_path = (DATA_PATH/"meta_data").with_suffix(UploadedFile.FileFormats.CSV) - spectral_file_path = (DATA_PATH / "spectral_data").with_suffix(UploadedFile.FileFormats.CSV) + array_file_path = (DATA_PATH / "array_data").with_suffix(UploadedFile.FileFormats.CSV) new_meta_data_path = tmp_path / meta_data_path.with_suffix(".blah") - new_spectral_file_path = tmp_path / spectral_file_path.with_suffix(".blah") + new_array_file_path = tmp_path / array_file_path.with_suffix(".blah") shutil.copyfile(meta_data_path, new_meta_data_path) - shutil.copyfile(spectral_file_path, new_spectral_file_path) + shutil.copyfile(array_file_path, new_array_file_path) - with new_meta_data_path.open(mode="rb") as meta_data, new_spectral_file_path.open(mode="rb") as spectral_data: + with new_meta_data_path.open(mode="rb") as meta_data, new_array_file_path.open(mode="rb") as array_data: data_upload = UploadedFile(meta_data_file=django.core.files.File(meta_data, name=new_meta_data_path.name), - spectral_data_file=django.core.files.File(spectral_data, - name=new_spectral_file_path.name), + array_data_file=django.core.files.File(array_data, + name=new_array_file_path.name), center=center) with pytest.raises(ValidationError, match="Allowed extensions"): data_upload.full_clean() @@ -606,8 +606,8 @@ def test_get_center(centers, mock_data_from_files): for bio_sample in visit.bio_sample.all(): assert get_center(bio_sample) is center - for spectral_data in bio_sample.spectral_data.all(): - assert get_center(spectral_data) is center + for array_data in bio_sample.array_data.all(): + assert get_center(array_data) is center for observation in Observation.objects.filter(visit__patient__center=center): assert get_center(observation) == center diff --git a/biodb/apps/uploader/tests/test_qc_models.py b/biodb/apps/uploader/tests/test_qc_models.py index 8d84c6f..6cc563d 100644 --- a/biodb/apps/uploader/tests/test_qc_models.py +++ b/biodb/apps/uploader/tests/test_qc_models.py @@ -6,7 +6,7 @@ from django.core.management import call_command from django.utils.module_loading import import_string -from uploader.models import QCAnnotation, QCAnnotator, SpectralData +from uploader.models import QCAnnotation, QCAnnotator, ArrayData import biodb.qc.qcfilter import biodb.util @@ -35,8 +35,8 @@ def test_unique_annotator(self, qcannotators): def test_new_annotation(self, qcannotators, mock_data_from_files): annotator = QCAnnotator.objects.get(name="sum") - spectral_data = SpectralData.objects.all()[0] - annotation = QCAnnotation(annotator=annotator, spectral_data=spectral_data) + array_data = ArrayData.objects.all()[0] + annotation = QCAnnotation(annotator=annotator, array_data=array_data) assert annotation.value is None annotation.full_clean() annotation.save() @@ -78,7 +78,7 @@ def test_annotation_get_value(self, qcannotators): assert annotation.get_value() == 3.14 @pytest.mark.parametrize("mock_data_from_files", [True], indirect=True) # AUTO_ANNOTATE = True - def test_auto_annotate_with_new_spectral_data(self, qcannotators, mock_data_from_files): + def test_auto_annotate_with_new_array_data(self, qcannotators, mock_data_from_files): for expected_results, annotation in zip(self.expected_sum_results, QCAnnotation.objects.all()): assert pytest.approx(annotation.get_value()) == expected_results @@ -98,22 +98,22 @@ def test_auto_annotate_with_new_default_annotator(self, monkeypatch, mock_data_f assert pytest.approx(annotation.get_value()) == expected_results def test_empty_get_annotators(self, mock_data_from_files): - for data in SpectralData.objects.all(): + for data in ArrayData.objects.all(): assert len(data.get_annotators()) == 0 @pytest.mark.parametrize("mock_data_from_files", [True], indirect=True) # AUTO_ANNOTATE = True def test_get_annotators(self, qcannotators, mock_data_from_files): - for data in SpectralData.objects.all(): + for data in ArrayData.objects.all(): assert len(data.get_annotators()) == 1 @pytest.mark.parametrize("mock_data_from_files", [True], indirect=True) # AUTO_ANNOTATE = True def test_get_zero_unrun_annotators(self, qcannotators, mock_data_from_files): - for data in SpectralData.objects.all(): + for data in ArrayData.objects.all(): assert len(data.get_unrun_annotators()) == 0 @pytest.mark.parametrize("mock_data_from_files", [True], indirect=True) # AUTO_ANNOTATE = True def test_get_unrun_annotators(self, monkeypatch, qcannotators, mock_data_from_files): - for data in SpectralData.objects.all(): + for data in ArrayData.objects.all(): assert len(data.get_unrun_annotators()) == 0 monkeypatch.setattr(settings, "RUN_DEFAULT_ANNOTATORS_WHEN_SAVED", False) @@ -124,12 +124,12 @@ def test_get_unrun_annotators(self, monkeypatch, qcannotators, mock_data_from_fi annotator.full_clean() annotator.save() - for data in SpectralData.objects.all(): + for data in ArrayData.objects.all(): assert len(data.get_unrun_annotators()) == 1 @pytest.mark.parametrize("mock_data_from_files", [True], indirect=True) # AUTO_ANNOTATE = True def test_get_new_unrun_annotators(self, monkeypatch, qcannotators, mock_data_from_files): - for data in SpectralData.objects.all(): + for data in ArrayData.objects.all(): assert len(data.get_unrun_annotators()) == 0 monkeypatch.setattr(settings, "RUN_DEFAULT_ANNOTATORS_WHEN_SAVED", True) @@ -140,7 +140,7 @@ def test_get_new_unrun_annotators(self, monkeypatch, qcannotators, mock_data_fro annotator.full_clean() annotator.save() - for data in SpectralData.objects.all(): + for data in ArrayData.objects.all(): assert len(data.get_unrun_annotators()) == 0 def test_management_command_no_annotators(self, db): @@ -151,7 +151,7 @@ def test_management_command_no_annotators(self, db): def test_management_command_no_data(self, qcannotators): out = StringIO() call_command("run_qc_annotators", stdout=out) - assert "No SpectralData exists to annotate." in out.getvalue() + assert "No ArrayData exists to annotate." in out.getvalue() def test_management_command(self, mock_data_from_files, qcannotators): for annotation in QCAnnotation.objects.all(): @@ -212,5 +212,5 @@ def test_no_file_validation(self, qcannotators): def test_no_file_related_error(self, qcannotators): """ Test that a validation error is raised rather than any other python exception which would indicate a bug. """ annotation = QCAnnotation(annotator=QCAnnotator.objects.get(name="sum")) - with pytest.raises(QCAnnotation.spectral_data.RelatedObjectDoesNotExist): + with pytest.raises(QCAnnotation.array_data.RelatedObjectDoesNotExist): annotation.save() diff --git a/biodb/apps/uploader/tests/test_uploader_admin.py b/biodb/apps/uploader/tests/test_uploader_admin.py index a1340e6..2b6a2a3 100644 --- a/biodb/apps/uploader/tests/test_uploader_admin.py +++ b/biodb/apps/uploader/tests/test_uploader_admin.py @@ -17,7 +17,7 @@ User = get_user_model() -SKIP_MODELS = [uploader.models.BioSampleType, uploader.models.SpectraMeasurementType] +SKIP_MODELS = [uploader.models.BioSampleType, uploader.models.ArrayMeasurementType] uploader_models = [] @@ -207,14 +207,14 @@ def test_non_form_field_validation(self, mock_data_from_files): c.force_login(user) meta_data_path = (DATA_PATH / "meta_data").with_suffix(uploader.models.UploadedFile.FileFormats.XLSX) - spectral_file_path = (DATA_PATH / "spectral_data").with_suffix(uploader.models.UploadedFile.FileFormats.XLSX) + array_file_path = (DATA_PATH / "array_data").with_suffix(uploader.models.UploadedFile.FileFormats.XLSX) with meta_data_path.open(mode="rb") as meta_data: - with spectral_file_path.open(mode="rb") as spectral_data: + with array_file_path.open(mode="rb") as array_data: meta_data_file = django.core.files.File(meta_data, name=meta_data_path.name) - spectral_data_file = django.core.files.File(spectral_data, name=spectral_file_path.name) + array_data_file = django.core.files.File(array_data, name=array_file_path.name) response = c.post("/data/uploader/uploadedfile/add/", follow=True, data={"meta_data_file": meta_data_file, - "spectral_data_file": spectral_data_file, + "array_data_file": array_data_file, "center": user.center.pk}) assert response.status_code == 200 diff --git a/biodb/qc/qcfilter.py b/biodb/qc/qcfilter.py index 00530d6..979553a 100644 --- a/biodb/qc/qcfilter.py +++ b/biodb/qc/qcfilter.py @@ -9,11 +9,11 @@ class QCValidationError(Exception): class QcFilter(ABC): @abstractmethod - def run(self, spectral_data): + def run(self, array_data): """ Implement this method to return the actual annotation value(s). - param: spectral_data - uploader.models.SpectralData + param: array_data - uploader.models.ArrayData Raises QCValidationError. """ @@ -21,19 +21,19 @@ def run(self, spectral_data): class QcSum(QcFilter): - def run(self, spectral_data: "SpectralData"): # noqa: F821 - data = spectral_data.get_spectral_data() + def run(self, array_data: "ArrayData"): # noqa: F821 + data = array_data.get_array_data() res = np.sum(data.intensity) return res class QcTestDummyTrue(QcFilter): """ For testing purposes only. """ - def run(self, spectral_data): + def run(self, array_data): return True class QcTestDummyFalse(QcFilter): """ For testing purposes only. """ - def run(self, spectral_data): + def run(self, array_data): return False diff --git a/biodb/qc/qcmanager.py b/biodb/qc/qcmanager.py index 86c71a3..9411953 100644 --- a/biodb/qc/qcmanager.py +++ b/biodb/qc/qcmanager.py @@ -3,7 +3,7 @@ from django.conf import settings from biodb.qc.qcfilter import QcFilter, QCValidationError -from uploader.models import SpectralData +from uploader.models import ArrayData log = logging.getLogger() @@ -35,7 +35,7 @@ def validator(self, value) -> None: # NOTE: validator vs validatorS is intentio self._validators[name] = filter - def validate(self, data: SpectralData) -> dict: + def validate(self, data: ArrayData) -> dict: results = {} for name, filter in self.validators.items(): try: diff --git a/biodb/settings/base.py b/biodb/settings/base.py index 39b3f9e..f340eb7 100644 --- a/biodb/settings/base.py +++ b/biodb/settings/base.py @@ -300,7 +300,7 @@ # NOTE: The following two settings don't actually belong to explorer. -# Include the spectral data files, if present in query results, for download as zip file. +# Include the array data files, if present in query results, for download as zip file. EXPLORER_DATA_EXPORTERS_INCLUDE_DATA_FILES = True # Exhaustively scan query result values for relevant filepaths to collect data files. Does nothing when # EXPLORER_DATA_EXPORTERS_INCLUDE_DATA_FILES == False. @@ -308,13 +308,13 @@ # Custom settings: -# Automatically run "default" annotators when new spectral data is saved. Note: Annotators are always run when new +# Automatically run "default" annotators when new array data is saved. Note: Annotators are always run when new # annotations are explicitly created and saved regardless of the below setting. AUTO_ANNOTATE = True -# Run newly added/updated annotator on all spectral data if annotator.default is True. +# Run newly added/updated annotator on all array data if annotator.default is True. # WARNING: This may be time-consuming if the annotators takes a while to run and there are a lot of -# spectral data samples in the database. +# array data samples in the database. RUN_DEFAULT_ANNOTATORS_WHEN_SAVED = False # Disable this class for now as #69 made it obsolete, however, there's a very good chance it will be needed diff --git a/biodb/util.py b/biodb/util.py index 3b4f47a..c46cf39 100644 --- a/biodb/util.py +++ b/biodb/util.py @@ -51,7 +51,7 @@ def to_bool(value): raise ValueError(f"Bool aliases are '{TRUE}|{FALSE}', not '{value}'") -def mock_bulk_spectral_data(path=Path.home(), +def mock_bulk_array_data(path=Path.home(), max_wavelength=4000, min_wavelength=651, n_bins=1798, @@ -62,8 +62,8 @@ def mock_bulk_spectral_data(path=Path.home(), data.index.name = settings.BULK_UPLOAD_INDEX_COLUMN_NAME data.index += 1 # Make index 1 based. - data.to_excel(path / "spectral_data.xlsx") - data.to_csv(path / "spectral_data.csv") + data.to_excel(path / "array_data.xlsx") + data.to_csv(path / "array_data.csv") return data From f70539bef1e5e999c58c1e6cbcd913671cfad9cb Mon Sep 17 00:00:00 2001 From: James Noss Date: Wed, 29 May 2024 22:44:57 -0400 Subject: [PATCH 3/5] Fix tests (and other stuff) Signed-off-by: James Noss --- biodb/apps/catalog/tests/conftest.py | 8 +-- biodb/apps/uploader/admin.py | 24 ------- ...ttypes.json => arraymeasurementtypes.json} | 4 +- biodb/apps/uploader/fixtures/instruments.json | 10 +-- .../apps/uploader/fixtures/qcannotators.json | 2 +- biodb/apps/uploader/fixtures/test_data.json | 64 +++++++----------- .../apps/uploader/migrations/0001_initial.py | 2 +- biodb/apps/uploader/models.py | 7 +- biodb/apps/uploader/tests/conftest.py | 12 ++-- .../{spectral_data.csv => array_data.csv} | 0 .../{spectral_data.jsonl => array_data.jsonl} | 0 .../{spectral_data.xlsx => array_data.xlsx} | Bin biodb/apps/uploader/tests/data/meta_data.csv | 24 +++---- .../apps/uploader/tests/data/meta_data.jsonl | 20 +++--- biodb/apps/uploader/tests/data/meta_data.xlsx | Bin 11859 -> 11713 bytes biodb/apps/uploader/tests/test_io.py | 8 +-- .../tests/test_management_commands.py | 6 +- biodb/apps/uploader/tests/test_models.py | 26 +++---- .../uploader/tests/test_uploader_admin.py | 6 +- biodb/apps/user/fixtures/centers.json | 8 +-- biodb/apps/user/tests/test_center.py | 4 +- 21 files changed, 94 insertions(+), 141 deletions(-) rename biodb/apps/uploader/fixtures/{spectrameasurementtypes.json => arraymeasurementtypes.json} (66%) rename biodb/apps/uploader/tests/data/{spectral_data.csv => array_data.csv} (100%) rename biodb/apps/uploader/tests/data/{spectral_data.jsonl => array_data.jsonl} (100%) rename biodb/apps/uploader/tests/data/{spectral_data.xlsx => array_data.xlsx} (100%) diff --git a/biodb/apps/catalog/tests/conftest.py b/biodb/apps/catalog/tests/conftest.py index 2a78eb9..7a5fe7f 100644 --- a/biodb/apps/catalog/tests/conftest.py +++ b/biodb/apps/catalog/tests/conftest.py @@ -5,7 +5,7 @@ from catalog.models import Dataset from uploader.tests.conftest import bio_sample_types, centers, instruments, mock_data_from_files, observables, \ - SimpleQueryFactory, spectra_measurement_types, sql_views, mock_data # noqa: F401 + SimpleQueryFactory, array_measurement_types, sql_views, mock_data # noqa: F401 from user.models import Center as UserCenter @@ -23,7 +23,7 @@ def staffuser(centers): # noqa: F811 return User.objects.create(username="staff", email="staff@jhu.edu", password="secret", - center=UserCenter.objects.get(name="jhu"), + center=UserCenter.objects.get(name="JHU"), is_staff=True, is_superuser=False) @@ -33,7 +33,7 @@ def cataloguser(centers): # noqa: F811 return User.objects.create(username="analyst", email="analyst@jhu.edu", password="secret", - center=UserCenter.objects.get(name="jhu"), + center=UserCenter.objects.get(name="JHU"), is_staff=True, is_superuser=False, is_catalogviewer=True) @@ -44,7 +44,7 @@ def superuser(centers): # noqa: F811 return User.objects.create(username="admin", email="admin@jhu.edu", password="secret", - center=UserCenter.objects.get(name="jhu"), + center=UserCenter.objects.get(name="JHU"), is_staff=True, is_superuser=True) diff --git a/biodb/apps/uploader/admin.py b/biodb/apps/uploader/admin.py index 4264d9c..d786d10 100644 --- a/biodb/apps/uploader/admin.py +++ b/biodb/apps/uploader/admin.py @@ -163,18 +163,6 @@ class InstrumentAdmin(RestrictedByCenterMixin, ModelAdmin): "center"] } ), - ( - "Spectrometer", - { - "fields": ["spectrometer_manufacturer", "spectrometer_model", "spectrometer_serial_number"], - } - ), - ( - "Laser", - { - "fields": ["laser_manufacturer", "laser_model", "laser_serial_number"], - } - ), ( "More Details", { @@ -468,8 +456,6 @@ class ArrayDataMixin: { "fields": ["measurement_id", "measurement_type", - "atr_crystal", - "n_coadditions", "acquisition_time", "resolution", "power", @@ -479,16 +465,6 @@ class ArrayDataMixin: "date"], } ), - ( - "SERS Details", - { - "classes": ["collapse"], - "fields": ["sers_description", - "sers_particle_material", - "sers_particle_size", - "sers_particle_concentration"], - } - ), ( "More Details", { diff --git a/biodb/apps/uploader/fixtures/spectrameasurementtypes.json b/biodb/apps/uploader/fixtures/arraymeasurementtypes.json similarity index 66% rename from biodb/apps/uploader/fixtures/spectrameasurementtypes.json rename to biodb/apps/uploader/fixtures/arraymeasurementtypes.json index ecd4a2a..bcae8f3 100644 --- a/biodb/apps/uploader/fixtures/spectrameasurementtypes.json +++ b/biodb/apps/uploader/fixtures/arraymeasurementtypes.json @@ -1,9 +1,9 @@ [ { - "model": "uploader.SpectraMeasurementType", + "model": "uploader.ArrayMeasurementType", "pk": 1, "fields": { - "name": "atr-ftir", + "name": "magic", "created_at": "2013-09-06T00:00:00+00:00", "updated_at": "2013-09-06T00:00:00+00:00" } diff --git a/biodb/apps/uploader/fixtures/instruments.json b/biodb/apps/uploader/fixtures/instruments.json index ee81010..b5cf554 100644 --- a/biodb/apps/uploader/fixtures/instruments.json +++ b/biodb/apps/uploader/fixtures/instruments.json @@ -4,15 +4,9 @@ "pk": "4205d8ac-90c1-4529-90b2-6751f665c403", "fields": { "cid": "", - "manufacturer": "Agilent", - "model": "Cary 630", + "manufacturer": "JHU", + "model": "Analyzer1000", "serial_number": "", - "spectrometer_manufacturer": "", - "spectrometer_model": "", - "spectrometer_serial_number": "", - "laser_manufacturer": "", - "laser_model": "", - "laser_serial_number": "", "created_at": "2013-09-06T00:00:00+00:00", "updated_at": "2013-09-06T00:00:00+00:00" } diff --git a/biodb/apps/uploader/fixtures/qcannotators.json b/biodb/apps/uploader/fixtures/qcannotators.json index bef3cd3..c7065e1 100644 --- a/biodb/apps/uploader/fixtures/qcannotators.json +++ b/biodb/apps/uploader/fixtures/qcannotators.json @@ -5,7 +5,7 @@ "fields": { "name": "sum", "fully_qualified_class_name": "biodb.qc.qcfilter.QcSum", - "description": "Sum of the spectral data (total flux)", + "description": "Sum of the array data (total flux)", "default": true, "value_type": "FLOAT", "created_at": "2013-09-06T00:00:00+00:00", diff --git a/biodb/apps/uploader/fixtures/test_data.json b/biodb/apps/uploader/fixtures/test_data.json index d1c010a..be63088 100644 --- a/biodb/apps/uploader/fixtures/test_data.json +++ b/biodb/apps/uploader/fixtures/test_data.json @@ -3175,15 +3175,9 @@ "pk": "4205d8ac-90c1-4529-90b2-6751f665c403", "fields": { "cid": "", - "manufacturer": "Agilent", - "model": "Cary 630", + "manufacturer": "JHU", + "model": "Analyzer1000", "serial_number": "", - "spectrometer_manufacturer": "", - "spectrometer_model": "", - "spectrometer_serial_number": "", - "laser_manufacturer": "", - "laser_model": "", - "laser_serial_number": "", "created_at": "2013-09-06T00:00:00+00:00", "updated_at": "2013-09-06T00:00:00+00:00" } @@ -3328,16 +3322,16 @@ } }, { - "model": "uploader.spectrameasurementtype", + "model": "uploader.arraymeasurementtype", "pk": 1, "fields": { - "name": "atr-ftir", + "name": "magic", "created_at": "2013-09-06T00:00:00+00:00", "updated_at": "2013-09-06T00:00:00+00:00" } }, { - "model": "uploader.spectraldata", + "model": "uploader.arraydata", "pk": 1117, "fields": { "created_at": "2013-09-06T00:00:00+00:00", @@ -3346,13 +3340,12 @@ "bio_sample": 1117, "measurement_type": 1, "acquisition_time": null, - "n_coadditions": 32, "resolution": null, - "data": "spectral_data/class_uploader.models_7GOLfEP.csv" + "data": "array_data/class_uploader.models_7GOLfEP.csv" } }, { - "model": "uploader.spectraldata", + "model": "uploader.arraydata", "pk": 1118, "fields": { "created_at": "2013-09-06T00:00:00+00:00", @@ -3361,13 +3354,12 @@ "bio_sample": 1118, "measurement_type": 1, "acquisition_time": null, - "n_coadditions": 32, "resolution": null, - "data": "spectral_data/class_uploader.models_02z3Zqt.csv" + "data": "array_data/class_uploader.models_02z3Zqt.csv" } }, { - "model": "uploader.spectraldata", + "model": "uploader.arraydata", "pk": 1119, "fields": { "created_at": "2013-09-06T00:00:00+00:00", @@ -3376,13 +3368,12 @@ "bio_sample": 1119, "measurement_type": 1, "acquisition_time": null, - "n_coadditions": 32, "resolution": null, - "data": "spectral_data/class_uploader.models_y6CI6Qs.csv" + "data": "array_data/class_uploader.models_y6CI6Qs.csv" } }, { - "model": "uploader.spectraldata", + "model": "uploader.arraydata", "pk": 1120, "fields": { "created_at": "2013-09-06T00:00:00+00:00", @@ -3391,13 +3382,12 @@ "bio_sample": 1120, "measurement_type": 1, "acquisition_time": null, - "n_coadditions": 32, "resolution": null, - "data": "spectral_data/class_uploader.models_8bK4evK.csv" + "data": "array_data/class_uploader.models_8bK4evK.csv" } }, { - "model": "uploader.spectraldata", + "model": "uploader.arraydata", "pk": 1121, "fields": { "created_at": "2013-09-06T00:00:00+00:00", @@ -3406,13 +3396,12 @@ "bio_sample": 1121, "measurement_type": 1, "acquisition_time": null, - "n_coadditions": 32, "resolution": null, - "data": "spectral_data/class_uploader.models_83JcnDj.csv" + "data": "array_data/class_uploader.models_83JcnDj.csv" } }, { - "model": "uploader.spectraldata", + "model": "uploader.arraydata", "pk": 1122, "fields": { "created_at": "2013-09-06T00:00:00+00:00", @@ -3421,13 +3410,12 @@ "bio_sample": 1122, "measurement_type": 1, "acquisition_time": null, - "n_coadditions": 32, "resolution": null, - "data": "spectral_data/class_uploader.models_oTQYhIJ.csv" + "data": "array_data/class_uploader.models_oTQYhIJ.csv" } }, { - "model": "uploader.spectraldata", + "model": "uploader.arraydata", "pk": 1123, "fields": { "created_at": "2013-09-06T00:00:00+00:00", @@ -3436,13 +3424,12 @@ "bio_sample": 1123, "measurement_type": 1, "acquisition_time": null, - "n_coadditions": 32, "resolution": null, - "data": "spectral_data/class_uploader.models_n8JSA1k.csv" + "data": "array_data/class_uploader.models_n8JSA1k.csv" } }, { - "model": "uploader.spectraldata", + "model": "uploader.arraydata", "pk": 1124, "fields": { "created_at": "2013-09-06T00:00:00+00:00", @@ -3451,13 +3438,12 @@ "bio_sample": 1124, "measurement_type": 1, "acquisition_time": null, - "n_coadditions": 32, "resolution": null, - "data": "spectral_data/class_uploader.models_xsCNQjJ.csv" + "data": "array_data/class_uploader.models_xsCNQjJ.csv" } }, { - "model": "uploader.spectraldata", + "model": "uploader.arraydata", "pk": 1125, "fields": { "created_at": "2013-09-06T00:00:00+00:00", @@ -3466,13 +3452,12 @@ "bio_sample": 1125, "measurement_type": 1, "acquisition_time": null, - "n_coadditions": 32, "resolution": null, - "data": "spectral_data/class_uploader.models_QkF6GzX.csv" + "data": "array_data/class_uploader.models_QkF6GzX.csv" } }, { - "model": "uploader.spectraldata", + "model": "uploader.arraydata", "pk": 1126, "fields": { "created_at": "2013-09-06T00:00:00+00:00", @@ -3481,9 +3466,8 @@ "bio_sample": 1126, "measurement_type": 1, "acquisition_time": null, - "n_coadditions": 32, "resolution": null, - "data": "spectral_data/class_uploader.models_qrNWV5h.csv" + "data": "array_data/class_uploader.models_qrNWV5h.csv" } } ] diff --git a/biodb/apps/uploader/migrations/0001_initial.py b/biodb/apps/uploader/migrations/0001_initial.py index 61cb5eb..d26e193 100644 --- a/biodb/apps/uploader/migrations/0001_initial.py +++ b/biodb/apps/uploader/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.11 on 2024-05-30 01:49 +# Generated by Django 4.2.11 on 2024-05-30 02:14 import django.core.validators from django.db import migrations, models diff --git a/biodb/apps/uploader/models.py b/biodb/apps/uploader/models.py index 9fe372e..7d5735e 100644 --- a/biodb/apps/uploader/models.py +++ b/biodb/apps/uploader/models.py @@ -599,8 +599,7 @@ class Meta: @classmethod def parse_fields_from_pandas_series(cls, series): """ Parse the pandas series for field values returning a dict. """ - return dict(spectrometer__iexact=get_field_value(series, cls, "spectrometer"), - atr_crystal__iexact=get_field_value(series, cls, "atr_crystal")) + return {} def __str__(self): return f"{self.manufacturer}_{self.model}_{self.id}" @@ -868,7 +867,7 @@ def annotate(self, annotator=None, force=False) -> list: return annotation = self.qc_annotation.get(annotator=annotator) else: - annotation = QCAnnotation(annotator=annotator, arary_data=self) + annotation = QCAnnotation(annotator=annotator, array_data=self) return [annotation.run()] annotations = [] @@ -1042,7 +1041,7 @@ def sql(cls): join bio_sample bs on bs.visit_id=v.id join bio_sample_type bst on bst.id=bs.sample_type_id join array_data sd on sd.bio_sample_id=bs.id - join spectra_measurement_type smt on smt.id=sd.measurement_type_id + join array_measurement_type smt on smt.id=sd.measurement_type_id join instrument i on i.id=sd.instrument_id left outer join v_visit_observations vs on vs.visit_id=v.id """ # nosec B608 diff --git a/biodb/apps/uploader/tests/conftest.py b/biodb/apps/uploader/tests/conftest.py index 998e8b8..594046a 100644 --- a/biodb/apps/uploader/tests/conftest.py +++ b/biodb/apps/uploader/tests/conftest.py @@ -79,7 +79,7 @@ class Meta: def django_request(center): request = RequestFactory() user = UserFactory() - user.center = UserCenter.objects.get(name="jhu") + user.center = UserCenter.objects.get(name="JHU") request.user = user return request @@ -93,7 +93,7 @@ def centers(django_db_blocker): @pytest.fixture(scope="function") def center(centers): - return Center.objects.get(name="jhu") + return Center.objects.get(name="JHU") @pytest.fixture(scope="function") @@ -109,9 +109,9 @@ def bio_sample_types(django_db_blocker): @pytest.fixture(scope="function") -def spectra_measurement_types(django_db_blocker): +def array_measurement_types(django_db_blocker): with django_db_blocker.unblock(): - call_command('loaddata', "--database=bsr", 'spectrameasurementtypes.json') + call_command('loaddata', "--database=bsr", 'arraymeasurementtypes.json') @pytest.fixture(scope="function") @@ -163,7 +163,7 @@ def bulk_upload(): name=meta_data_path.name), array_data_file=django.core.files.File(array_data, name=array_file_path.name), - center=Center.objects.get(name="jhu")) + center=Center.objects.get(name="JHU")) data_upload.clean() data_upload.save() @@ -177,7 +177,7 @@ def mock_data_from_files(request, django_db_blocker, instruments, bio_sample_types, - spectra_measurement_types): + array_measurement_types): # patch MEDIA_ROOT media_root = request.node.get_closest_marker("media_root") if media_root: diff --git a/biodb/apps/uploader/tests/data/spectral_data.csv b/biodb/apps/uploader/tests/data/array_data.csv similarity index 100% rename from biodb/apps/uploader/tests/data/spectral_data.csv rename to biodb/apps/uploader/tests/data/array_data.csv diff --git a/biodb/apps/uploader/tests/data/spectral_data.jsonl b/biodb/apps/uploader/tests/data/array_data.jsonl similarity index 100% rename from biodb/apps/uploader/tests/data/spectral_data.jsonl rename to biodb/apps/uploader/tests/data/array_data.jsonl diff --git a/biodb/apps/uploader/tests/data/spectral_data.xlsx b/biodb/apps/uploader/tests/data/array_data.xlsx similarity index 100% rename from biodb/apps/uploader/tests/data/spectral_data.xlsx rename to biodb/apps/uploader/tests/data/array_data.xlsx diff --git a/biodb/apps/uploader/tests/data/meta_data.csv b/biodb/apps/uploader/tests/data/meta_data.csv index 1c717a5..51f47a4 100644 --- a/biodb/apps/uploader/tests/data/meta_data.csv +++ b/biodb/apps/uploader/tests/data/meta_data.csv @@ -1,12 +1,12 @@ -Patient CID,RESULT RT-qPCR,CTs (Gene N),CTs (Gene ORF),Gender,Age,Days observed,Fever,Dyspnoea,O2 Sat <95%,Cough,Coryza,Odinophagy,Diarrhea,Nausea,Headache,Weakness,ANOSMIA,MYALGIA,Lack of Appetite,Vomiting,Suspicious contact,CHRONIC PULMONARY DISEASE INCLUDING ASTHMA,CARDIOVASCULAR DISEASE INCLUDING HYPERTENSION,DIABETES,CHRONIC OR NEUROMUSCULAR NEUROLOGICAL DISEASE,Measurement type,Instrument,ATR Crystal,Acquisition time [s],Number of coadditions,Resolution [cm-1],Sample Type,Sample Processing,Freezing Temperature,Thawing time -,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -fd39991e-8791-4a17-91de-5fee78236d6d,Negative,,,Man,21,7,No,No,No,Yes,Yes,No,Yes,No,Yes,No,Yes,No,No,No,Yes,No,No,No,No,ATR-FTIR,4205d8ac-90c1-4529-90b2-6751f665c403,ZnSe,,32,,Pharyngeal Swab,None,, -f52d82f3-1fb4-4734-b0d3-56665153243c,Negative,,,Man,97,1,No,No,No,Yes,Yes,No,Yes,No,Yes,No,Yes,No,No,No,Yes,No,No,No,No,ATR-FTIR,4205d8ac-90c1-4529-90b2-6751f665c403,ZnSe,,32,,Pharyngeal Swab,None,, -d0ffda4a-6e60-4bff-8929-1c5b998c8d28,Negative,,,woman,6,2,Yes,No,No,Yes,Yes,No,No,No,Yes,Yes,No,Yes,No,No,No,No,No,No,No,ATR-FTIR,4205d8ac-90c1-4529-90b2-6751f665c403,ZnSe,,32,,Pharyngeal Swab,None,, -d0e2eef7-2fc2-41c1-b1a9-226a9686772d,Negative,,,woman,44,2,Yes,No,No,Yes,Yes,No,No,No,Yes,Yes,No,Yes,No,No,No,No,No,No,No,ATR-FTIR,4205d8ac-90c1-4529-90b2-6751f665c403,ZnSe,,32,,Pharyngeal Swab,None,, -e7defaff-ac0d-409d-a497-cebe94c4ce01,Negative,,,woman,55,3,No,No,No,Yes,Yes,No,No,No,No,No,No,No,No,No,No,No,No,No,No,ATR-FTIR,4205d8ac-90c1-4529-90b2-6751f665c403,ZnSe,,32,,Pharyngeal Swab,None,, -2c8e8f01-7e2b-4ff6-9be6-60fa4914cd24,Positive,24.8,24.8,woman,22,Unknown,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,ATR-FTIR,4205d8ac-90c1-4529-90b2-6751f665c403,ZnSe,,32,,Pharyngeal Swab,None,, -2204c553-1098-42e6-87cd-8baeedfed672,Positive,18.18,18.18,Man,17,3,Yes,No,No,Yes,No,Yes,No,No,No,Yes,No,No,No,No,No,No,No,No,No,ATR-FTIR,4205d8ac-90c1-4529-90b2-6751f665c403,ZnSe,,32,,Pharyngeal Swab,None,, -651bb92d-74af-44d2-a3ca-ef3cbc70ee0c,Negative,,,Man,57,5,Yes,No,No,Yes,Yes,Yes,No,Yes,Yes,Yes,No,No,No,No,No,No,No,No,No,ATR-FTIR,4205d8ac-90c1-4529-90b2-6751f665c403,ZnSe,,32,,Pharyngeal Swab,None,, -34c9a874-1227-44d2-9274-3ff757e0fcff,Negative,,,Man,9,11,No,No,No,Yes,No,Yes,No,No,No,No,No,No,No,No,No,Yes,No,No,No,ATR-FTIR,4205d8ac-90c1-4529-90b2-6751f665c403,ZnSe,,32,,Pharyngeal Swab,None,, -4fd057b7-7078-4900-825c-28d145afa25e,Negative,,,prefer not to state,31,12,Yes,No,No,Yes,Yes,Yes,Yes,No,No,Yes,No,No,No,No,No,No,No,No,No,ATR-FTIR,4205d8ac-90c1-4529-90b2-6751f665c403,ZnSe,,32,,Pharyngeal Swab,None,, \ No newline at end of file +Patient CID,RESULT RT-qPCR,CTs (Gene N),CTs (Gene ORF),Gender,Age,Days observed,Fever,Dyspnoea,O2 Sat <95%,Cough,Coryza,Odinophagy,Diarrhea,Nausea,Headache,Weakness,ANOSMIA,MYALGIA,Lack of Appetite,Vomiting,Suspicious contact,CHRONIC PULMONARY DISEASE INCLUDING ASTHMA,CARDIOVASCULAR DISEASE INCLUDING HYPERTENSION,DIABETES,CHRONIC OR NEUROMUSCULAR NEUROLOGICAL DISEASE,Measurement type,Instrument,Acquisition time [s],Resolution [cm-1],Sample Type,Sample Processing,Freezing Temperature,Thawing time +,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +fd39991e-8791-4a17-91de-5fee78236d6d,Negative,,,Man,21,7,No,No,No,Yes,Yes,No,Yes,No,Yes,No,Yes,No,No,No,Yes,No,No,No,No,magic,4205d8ac-90c1-4529-90b2-6751f665c403,,,Pharyngeal Swab,None,, +f52d82f3-1fb4-4734-b0d3-56665153243c,Negative,,,Man,97,1,No,No,No,Yes,Yes,No,Yes,No,Yes,No,Yes,No,No,No,Yes,No,No,No,No,magic,4205d8ac-90c1-4529-90b2-6751f665c403,,,Pharyngeal Swab,None,, +d0ffda4a-6e60-4bff-8929-1c5b998c8d28,Negative,,,woman,6,2,Yes,No,No,Yes,Yes,No,No,No,Yes,Yes,No,Yes,No,No,No,No,No,No,No,magic,4205d8ac-90c1-4529-90b2-6751f665c403,,,Pharyngeal Swab,None,, +d0e2eef7-2fc2-41c1-b1a9-226a9686772d,Negative,,,woman,44,2,Yes,No,No,Yes,Yes,No,No,No,Yes,Yes,No,Yes,No,No,No,No,No,No,No,magic,4205d8ac-90c1-4529-90b2-6751f665c403,,,Pharyngeal Swab,None,, +e7defaff-ac0d-409d-a497-cebe94c4ce01,Negative,,,woman,55,3,No,No,No,Yes,Yes,No,No,No,No,No,No,No,No,No,No,No,No,No,No,magic,4205d8ac-90c1-4529-90b2-6751f665c403,,,Pharyngeal Swab,None,, +2c8e8f01-7e2b-4ff6-9be6-60fa4914cd24,Positive,24.8,24.8,woman,22,Unknown,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,No,magic,4205d8ac-90c1-4529-90b2-6751f665c403,,,Pharyngeal Swab,None,, +2204c553-1098-42e6-87cd-8baeedfed672,Positive,18.18,18.18,Man,17,3,Yes,No,No,Yes,No,Yes,No,No,No,Yes,No,No,No,No,No,No,No,No,No,magic,4205d8ac-90c1-4529-90b2-6751f665c403,,,Pharyngeal Swab,None,, +651bb92d-74af-44d2-a3ca-ef3cbc70ee0c,Negative,,,Man,57,5,Yes,No,No,Yes,Yes,Yes,No,Yes,Yes,Yes,No,No,No,No,No,No,No,No,No,magic,4205d8ac-90c1-4529-90b2-6751f665c403,,,Pharyngeal Swab,None,, +34c9a874-1227-44d2-9274-3ff757e0fcff,Negative,,,Man,9,11,No,No,No,Yes,No,Yes,No,No,No,No,No,No,No,No,No,Yes,No,No,No,magic,4205d8ac-90c1-4529-90b2-6751f665c403,,,Pharyngeal Swab,None,, +4fd057b7-7078-4900-825c-28d145afa25e,Negative,,,prefer not to state,31,12,Yes,No,No,Yes,Yes,Yes,Yes,No,No,Yes,No,No,No,No,No,No,No,No,No,magic,4205d8ac-90c1-4529-90b2-6751f665c403,,,Pharyngeal Swab,None,, \ No newline at end of file diff --git a/biodb/apps/uploader/tests/data/meta_data.jsonl b/biodb/apps/uploader/tests/data/meta_data.jsonl index d202d61..70ac341 100644 --- a/biodb/apps/uploader/tests/data/meta_data.jsonl +++ b/biodb/apps/uploader/tests/data/meta_data.jsonl @@ -1,10 +1,10 @@ -{"patient_cid":"fd39991e-8791-4a17-91de-5fee78236d6d","result rt-qpcr":"Negative","cts (gene n)":null,"cts (gene orf)":null,"gender":"Man","age":21.0,"days observed":7.0,"fever":false,"dyspnoea":false,"o2 sat <95%":false,"cough":true,"coryza":true,"odinophagy":false,"diarrhea":true,"nausea":false,"headache":true,"weakness":false,"anosmia":true,"myalgia":false,"lack of appetite":false,"vomiting":false,"suspicious contact":true,"chronic pulmonary disease including asthma":false,"cardiovascular disease including hypertension":false,"diabetes":false,"chronic or neuromuscular neurological disease":false,"Measurement type":"ATR-FTIR","Instrument":"4205d8ac-90c1-4529-90b2-6751f665c403","atr crystal":"ZnSe","acquisition time [s]":null,"number of coadditions":32.0,"resolution [cm-1]":null,"sample type":"Pharyngeal Swab","sample processing":null,"freezing temperature":null,"thawing time":null} -{"patient_cid":"f52d82f3-1fb4-4734-b0d3-56665153243c","result rt-qpcr":"Negative","cts (gene n)":null,"cts (gene orf)":null,"gender":"Man","age":97.0,"days observed":1.0,"fever":false,"dyspnoea":false,"o2 sat <95%":false,"cough":true,"coryza":true,"odinophagy":false,"diarrhea":true,"nausea":false,"headache":true,"weakness":false,"anosmia":true,"myalgia":false,"lack of appetite":false,"vomiting":false,"suspicious contact":true,"chronic pulmonary disease including asthma":false,"cardiovascular disease including hypertension":false,"diabetes":false,"chronic or neuromuscular neurological disease":false,"Measurement type":"ATR-FTIR","Instrument":"4205d8ac-90c1-4529-90b2-6751f665c403","atr crystal":"ZnSe","acquisition time [s]":null,"number of coadditions":32.0,"resolution [cm-1]":null,"sample type":"Pharyngeal Swab","sample processing":null,"freezing temperature":null,"thawing time":null} -{"patient_cid":"d0ffda4a-6e60-4bff-8929-1c5b998c8d28","result rt-qpcr":"Negative","cts (gene n)":null,"cts (gene orf)":null,"gender":"woman","age":6.0,"days observed":2.0,"fever":true,"dyspnoea":false,"o2 sat <95%":false,"cough":true,"coryza":true,"odinophagy":false,"diarrhea":false,"nausea":false,"headache":true,"weakness":true,"anosmia":false,"myalgia":true,"lack of appetite":false,"vomiting":false,"suspicious contact":false,"chronic pulmonary disease including asthma":false,"cardiovascular disease including hypertension":false,"diabetes":false,"chronic or neuromuscular neurological disease":false,"Measurement type":"ATR-FTIR","Instrument":"4205d8ac-90c1-4529-90b2-6751f665c403","atr crystal":"ZnSe","acquisition time [s]":null,"number of coadditions":32.0,"resolution [cm-1]":null,"sample type":"Pharyngeal Swab","sample processing":null,"freezing temperature":null,"thawing time":null} -{"patient_cid":"d0e2eef7-2fc2-41c1-b1a9-226a9686772d","result rt-qpcr":"Negative","cts (gene n)":null,"cts (gene orf)":null,"gender":"woman","age":44.0,"days observed":2.0,"fever":true,"dyspnoea":false,"o2 sat <95%":false,"cough":true,"coryza":true,"odinophagy":false,"diarrhea":false,"nausea":false,"headache":true,"weakness":true,"anosmia":false,"myalgia":true,"lack of appetite":false,"vomiting":false,"suspicious contact":false,"chronic pulmonary disease including asthma":false,"cardiovascular disease including hypertension":false,"diabetes":false,"chronic or neuromuscular neurological disease":false,"Measurement type":"ATR-FTIR","Instrument":"4205d8ac-90c1-4529-90b2-6751f665c403","atr crystal":"ZnSe","acquisition time [s]":null,"number of coadditions":32.0,"resolution [cm-1]":null,"sample type":"Pharyngeal Swab","sample processing":null,"freezing temperature":null,"thawing time":null} -{"patient_cid":"e7defaff-ac0d-409d-a497-cebe94c4ce01","result rt-qpcr":"Negative","cts (gene n)":null,"cts (gene orf)":null,"gender":"woman","age":55.0,"days observed":3.0,"fever":false,"dyspnoea":false,"o2 sat <95%":false,"cough":true,"coryza":true,"odinophagy":false,"diarrhea":false,"nausea":false,"headache":false,"weakness":false,"anosmia":false,"myalgia":false,"lack of appetite":false,"vomiting":false,"suspicious contact":false,"chronic pulmonary disease including asthma":false,"cardiovascular disease including hypertension":false,"diabetes":false,"chronic or neuromuscular neurological disease":false,"Measurement type":"ATR-FTIR","Instrument":"4205d8ac-90c1-4529-90b2-6751f665c403","atr crystal":"ZnSe","acquisition time [s]":null,"number of coadditions":32.0,"resolution [cm-1]":null,"sample type":"Pharyngeal Swab","sample processing":null,"freezing temperature":null,"thawing time":null} -{"patient_cid":"2c8e8f01-7e2b-4ff6-9be6-60fa4914cd24","result rt-qpcr":"Positive","cts (gene n)":24.8,"cts (gene orf)":24.8,"gender":"woman","age":22.0,"days observed":null,"fever":false,"dyspnoea":false,"o2 sat <95%":false,"cough":false,"coryza":false,"odinophagy":false,"diarrhea":false,"nausea":false,"headache":false,"weakness":false,"anosmia":false,"myalgia":false,"lack of appetite":false,"vomiting":false,"suspicious contact":false,"chronic pulmonary disease including asthma":false,"cardiovascular disease including hypertension":false,"diabetes":false,"chronic or neuromuscular neurological disease":false,"Measurement type":"ATR-FTIR","Instrument":"4205d8ac-90c1-4529-90b2-6751f665c403","atr crystal":"ZnSe","acquisition time [s]":null,"number of coadditions":32.0,"resolution [cm-1]":null,"sample type":"Pharyngeal Swab","sample processing":null,"freezing temperature":null,"thawing time":null} -{"patient_cid":"2204c553-1098-42e6-87cd-8baeedfed672","result rt-qpcr":"Positive","cts (gene n)":18.18,"cts (gene orf)":18.18,"gender":"man","age":17.0,"days observed":3.0,"fever":true,"dyspnoea":false,"o2 sat <95%":false,"cough":true,"coryza":false,"odinophagy":true,"diarrhea":false,"nausea":false,"headache":false,"weakness":true,"anosmia":false,"myalgia":false,"lack of appetite":false,"vomiting":false,"suspicious contact":false,"chronic pulmonary disease including asthma":false,"cardiovascular disease including hypertension":false,"diabetes":false,"chronic or neuromuscular neurological disease":false,"Measurement type":"ATR-FTIR","Instrument":"4205d8ac-90c1-4529-90b2-6751f665c403","atr crystal":"ZnSe","acquisition time [s]":null,"number of coadditions":32.0,"resolution [cm-1]":null,"sample type":"Pharyngeal Swab","sample processing":null,"freezing temperature":null,"thawing time":null} -{"patient_cid":"651bb92d-74af-44d2-a3ca-ef3cbc70ee0c","result rt-qpcr":"Negative","cts (gene n)":null,"cts (gene orf)":null,"gender":"MAN","age":57.0,"days observed":5.0,"fever":true,"dyspnoea":false,"o2 sat <95%":false,"cough":true,"coryza":true,"odinophagy":true,"diarrhea":false,"nausea":true,"headache":true,"weakness":true,"anosmia":false,"myalgia":false,"lack of appetite":false,"vomiting":false,"suspicious contact":false,"chronic pulmonary disease including asthma":false,"cardiovascular disease including hypertension":false,"diabetes":false,"chronic or neuromuscular neurological disease":false,"Measurement type":"ATR-FTIR","Instrument":"4205d8ac-90c1-4529-90b2-6751f665c403","atr crystal":"ZnSe","acquisition time [s]":null,"number of coadditions":32.0,"resolution [cm-1]":null,"sample type":"Pharyngeal Swab","sample processing":null,"freezing temperature":null,"thawing time":null} -{"patient_cid":"34c9a874-1227-44d2-9274-3ff757e0fcff","result rt-qpcr":"Negative","cts (gene n)":null,"cts (gene orf)":null,"gender":"Man","age":9.0,"days observed":11.0,"fever":false,"dyspnoea":false,"o2 sat <95%":false,"cough":true,"coryza":false,"odinophagy":true,"diarrhea":false,"nausea":false,"headache":false,"weakness":false,"anosmia":false,"myalgia":false,"lack of appetite":false,"vomiting":false,"suspicious contact":false,"chronic pulmonary disease including asthma":true,"cardiovascular disease including hypertension":false,"diabetes":false,"chronic or neuromuscular neurological disease":false,"Measurement type":"ATR-FTIR","Instrument":"4205d8ac-90c1-4529-90b2-6751f665c403","atr crystal":"ZnSe","acquisition time [s]":null,"number of coadditions":32.0,"resolution [cm-1]":null,"sample type":"Pharyngeal Swab","sample processing":null,"freezing temperature":null,"thawing time":null} -{"patient_cid":"4fd057b7-7078-4900-825c-28d145afa25e","result rt-qpcr":"Negative","cts (gene n)":null,"cts (gene orf)":null,"gender":"prefer not to state","age":31.0,"days observed":12.0,"fever":true,"dyspnoea":false,"o2 sat <95%":false,"cough":true,"coryza":true,"odinophagy":true,"diarrhea":true,"nausea":false,"headache":false,"weakness":true,"anosmia":false,"myalgia":false,"lack of appetite":false,"vomiting":false,"suspicious contact":false,"chronic pulmonary disease including asthma":false,"cardiovascular disease including hypertension":false,"diabetes":false,"chronic or neuromuscular neurological disease":false,"Measurement type":"ATR-FTIR","Instrument":"4205d8ac-90c1-4529-90b2-6751f665c403","atr crystal":"ZnSe","acquisition time [s]":null,"number of coadditions":32.0,"resolution [cm-1]":null,"sample type":"Pharyngeal Swab","sample processing":null,"freezing temperature":null,"thawing time":null} +{"patient_cid":"fd39991e-8791-4a17-91de-5fee78236d6d","result rt-qpcr":"Negative","cts (gene n)":null,"cts (gene orf)":null,"gender":"Man","age":21.0,"days observed":7.0,"fever":false,"dyspnoea":false,"o2 sat <95%":false,"cough":true,"coryza":true,"odinophagy":false,"diarrhea":true,"nausea":false,"headache":true,"weakness":false,"anosmia":true,"myalgia":false,"lack of appetite":false,"vomiting":false,"suspicious contact":true,"chronic pulmonary disease including asthma":false,"cardiovascular disease including hypertension":false,"diabetes":false,"chronic or neuromuscular neurological disease":false,"Measurement type":"magic","Instrument":"4205d8ac-90c1-4529-90b2-6751f665c403","acquisition time [s]":null,"resolution [cm-1]":null,"sample type":"Pharyngeal Swab","sample processing":null,"freezing temperature":null,"thawing time":null} +{"patient_cid":"f52d82f3-1fb4-4734-b0d3-56665153243c","result rt-qpcr":"Negative","cts (gene n)":null,"cts (gene orf)":null,"gender":"Man","age":97.0,"days observed":1.0,"fever":false,"dyspnoea":false,"o2 sat <95%":false,"cough":true,"coryza":true,"odinophagy":false,"diarrhea":true,"nausea":false,"headache":true,"weakness":false,"anosmia":true,"myalgia":false,"lack of appetite":false,"vomiting":false,"suspicious contact":true,"chronic pulmonary disease including asthma":false,"cardiovascular disease including hypertension":false,"diabetes":false,"chronic or neuromuscular neurological disease":false,"Measurement type":"magic","Instrument":"4205d8ac-90c1-4529-90b2-6751f665c403","acquisition time [s]":null,"resolution [cm-1]":null,"sample type":"Pharyngeal Swab","sample processing":null,"freezing temperature":null,"thawing time":null} +{"patient_cid":"d0ffda4a-6e60-4bff-8929-1c5b998c8d28","result rt-qpcr":"Negative","cts (gene n)":null,"cts (gene orf)":null,"gender":"woman","age":6.0,"days observed":2.0,"fever":true,"dyspnoea":false,"o2 sat <95%":false,"cough":true,"coryza":true,"odinophagy":false,"diarrhea":false,"nausea":false,"headache":true,"weakness":true,"anosmia":false,"myalgia":true,"lack of appetite":false,"vomiting":false,"suspicious contact":false,"chronic pulmonary disease including asthma":false,"cardiovascular disease including hypertension":false,"diabetes":false,"chronic or neuromuscular neurological disease":false,"Measurement type":"magic","Instrument":"4205d8ac-90c1-4529-90b2-6751f665c403","acquisition time [s]":null,"resolution [cm-1]":null,"sample type":"Pharyngeal Swab","sample processing":null,"freezing temperature":null,"thawing time":null} +{"patient_cid":"d0e2eef7-2fc2-41c1-b1a9-226a9686772d","result rt-qpcr":"Negative","cts (gene n)":null,"cts (gene orf)":null,"gender":"woman","age":44.0,"days observed":2.0,"fever":true,"dyspnoea":false,"o2 sat <95%":false,"cough":true,"coryza":true,"odinophagy":false,"diarrhea":false,"nausea":false,"headache":true,"weakness":true,"anosmia":false,"myalgia":true,"lack of appetite":false,"vomiting":false,"suspicious contact":false,"chronic pulmonary disease including asthma":false,"cardiovascular disease including hypertension":false,"diabetes":false,"chronic or neuromuscular neurological disease":false,"Measurement type":"magic","Instrument":"4205d8ac-90c1-4529-90b2-6751f665c403","acquisition time [s]":null,"resolution [cm-1]":null,"sample type":"Pharyngeal Swab","sample processing":null,"freezing temperature":null,"thawing time":null} +{"patient_cid":"e7defaff-ac0d-409d-a497-cebe94c4ce01","result rt-qpcr":"Negative","cts (gene n)":null,"cts (gene orf)":null,"gender":"woman","age":55.0,"days observed":3.0,"fever":false,"dyspnoea":false,"o2 sat <95%":false,"cough":true,"coryza":true,"odinophagy":false,"diarrhea":false,"nausea":false,"headache":false,"weakness":false,"anosmia":false,"myalgia":false,"lack of appetite":false,"vomiting":false,"suspicious contact":false,"chronic pulmonary disease including asthma":false,"cardiovascular disease including hypertension":false,"diabetes":false,"chronic or neuromuscular neurological disease":false,"Measurement type":"magic","Instrument":"4205d8ac-90c1-4529-90b2-6751f665c403","acquisition time [s]":null,"resolution [cm-1]":null,"sample type":"Pharyngeal Swab","sample processing":null,"freezing temperature":null,"thawing time":null} +{"patient_cid":"2c8e8f01-7e2b-4ff6-9be6-60fa4914cd24","result rt-qpcr":"Positive","cts (gene n)":24.8,"cts (gene orf)":24.8,"gender":"woman","age":22.0,"days observed":null,"fever":false,"dyspnoea":false,"o2 sat <95%":false,"cough":false,"coryza":false,"odinophagy":false,"diarrhea":false,"nausea":false,"headache":false,"weakness":false,"anosmia":false,"myalgia":false,"lack of appetite":false,"vomiting":false,"suspicious contact":false,"chronic pulmonary disease including asthma":false,"cardiovascular disease including hypertension":false,"diabetes":false,"chronic or neuromuscular neurological disease":false,"Measurement type":"magic","Instrument":"4205d8ac-90c1-4529-90b2-6751f665c403","acquisition time [s]":null,"resolution [cm-1]":null,"sample type":"Pharyngeal Swab","sample processing":null,"freezing temperature":null,"thawing time":null} +{"patient_cid":"2204c553-1098-42e6-87cd-8baeedfed672","result rt-qpcr":"Positive","cts (gene n)":18.18,"cts (gene orf)":18.18,"gender":"man","age":17.0,"days observed":3.0,"fever":true,"dyspnoea":false,"o2 sat <95%":false,"cough":true,"coryza":false,"odinophagy":true,"diarrhea":false,"nausea":false,"headache":false,"weakness":true,"anosmia":false,"myalgia":false,"lack of appetite":false,"vomiting":false,"suspicious contact":false,"chronic pulmonary disease including asthma":false,"cardiovascular disease including hypertension":false,"diabetes":false,"chronic or neuromuscular neurological disease":false,"Measurement type":"magic","Instrument":"4205d8ac-90c1-4529-90b2-6751f665c403","acquisition time [s]":null,"resolution [cm-1]":null,"sample type":"Pharyngeal Swab","sample processing":null,"freezing temperature":null,"thawing time":null} +{"patient_cid":"651bb92d-74af-44d2-a3ca-ef3cbc70ee0c","result rt-qpcr":"Negative","cts (gene n)":null,"cts (gene orf)":null,"gender":"MAN","age":57.0,"days observed":5.0,"fever":true,"dyspnoea":false,"o2 sat <95%":false,"cough":true,"coryza":true,"odinophagy":true,"diarrhea":false,"nausea":true,"headache":true,"weakness":true,"anosmia":false,"myalgia":false,"lack of appetite":false,"vomiting":false,"suspicious contact":false,"chronic pulmonary disease including asthma":false,"cardiovascular disease including hypertension":false,"diabetes":false,"chronic or neuromuscular neurological disease":false,"Measurement type":"magic","Instrument":"4205d8ac-90c1-4529-90b2-6751f665c403","acquisition time [s]":null,"resolution [cm-1]":null,"sample type":"Pharyngeal Swab","sample processing":null,"freezing temperature":null,"thawing time":null} +{"patient_cid":"34c9a874-1227-44d2-9274-3ff757e0fcff","result rt-qpcr":"Negative","cts (gene n)":null,"cts (gene orf)":null,"gender":"Man","age":9.0,"days observed":11.0,"fever":false,"dyspnoea":false,"o2 sat <95%":false,"cough":true,"coryza":false,"odinophagy":true,"diarrhea":false,"nausea":false,"headache":false,"weakness":false,"anosmia":false,"myalgia":false,"lack of appetite":false,"vomiting":false,"suspicious contact":false,"chronic pulmonary disease including asthma":true,"cardiovascular disease including hypertension":false,"diabetes":false,"chronic or neuromuscular neurological disease":false,"Measurement type":"magic","Instrument":"4205d8ac-90c1-4529-90b2-6751f665c403","acquisition time [s]":null,"resolution [cm-1]":null,"sample type":"Pharyngeal Swab","sample processing":null,"freezing temperature":null,"thawing time":null} +{"patient_cid":"4fd057b7-7078-4900-825c-28d145afa25e","result rt-qpcr":"Negative","cts (gene n)":null,"cts (gene orf)":null,"gender":"prefer not to state","age":31.0,"days observed":12.0,"fever":true,"dyspnoea":false,"o2 sat <95%":false,"cough":true,"coryza":true,"odinophagy":true,"diarrhea":true,"nausea":false,"headache":false,"weakness":true,"anosmia":false,"myalgia":false,"lack of appetite":false,"vomiting":false,"suspicious contact":false,"chronic pulmonary disease including asthma":false,"cardiovascular disease including hypertension":false,"diabetes":false,"chronic or neuromuscular neurological disease":false,"Measurement type":"magic","Instrument":"4205d8ac-90c1-4529-90b2-6751f665c403","acquisition time [s]":null,"resolution [cm-1]":null,"sample type":"Pharyngeal Swab","sample processing":null,"freezing temperature":null,"thawing time":null} diff --git a/biodb/apps/uploader/tests/data/meta_data.xlsx b/biodb/apps/uploader/tests/data/meta_data.xlsx index 03d02cae80a2ea2281b50fbb794dc710df9019c2..d524aed6e5e93131667aa830e68845b0ec7f47f7 100644 GIT binary patch delta 5936 zcma)Aby(C*yIvZ}1(sU6L0a~uQAAooy1P4;l=z8A=d#iuT}p#=tx9*7w9;K7d0yY| zobNl=`RmLdGjq+%HTU(z{meYiq%q2)JaW2=_B z==H&uK-2FqTA#&*KR0aGl~G*g7{A338SM}?JBwaAD0CWc zxKno0DjR5c{L#zMeKn&XNI!^MDQh^0{7T@=pqc7nn2IUww3vB>h7Oe(nW zDCz66TkNxxE-Erh+B|iFy7W8fP^gC_QJ~%aC`<2X;(IHwgFJAU|YvSf8Ij3vJwKU2o zOLQ*aQBWhn7M~d(7y+mhODDui+NqK9U-UX8^+>yI-#@H%?B6ianm!M=Q7ST3&Ul83 zaYD@q>M)j?NO%2`nxD)0c8JTFL|{jR(xUtZ0r#rcBA)g1OJ3B8)Uw}RaJYSXw3}nA z?K{?A9ChPu-{agFFlT%ueCd5X7O=)F=)dyp&Mm$X9MrAmH6nPZ=9LAso0{fq2PE?` z9M)-N7wm~fsr^czh}llCxxR77yzYIM-?3`DZxfB3ZhT9KL|{Pb2C#+D@nQ{sbX7e9 zflMWlDpZWXVtl3mIQ)qFf{tf}16xLo)4H2I&gbYOeZy=I0V+DK>>x|~V#yc8X22yf zu56x;y1!39IqIHN;`RjX6_#m1i7=FNDT)&>go`=>7i=NrtSxU;av4*Ro%^}N?~B28 z`~49e&a1%{Q5aMpXK4fwlwRdd$%#fa>f?*+h8r({7Gnsh zdQ(NTKBrbjn9?Pw{Ci%??9gRa+NB7w6Um21c<*quA8|PlX3*3*QFd5zN*C`@)T{8d zNp|7Am+=Z8Tv{s-a4-fxz?=y|u7W5%u56a(&?Y~s0eq^0s0^^FnKZi7Ok~w@3FJK3Dhi-3b z&{k#^b{E>Tr!Qx{<6SQ8;>Fq@uFmf6>K?9tU;90rRRNX3PhlBxpAtIFfWSiFX890S zdGF$5y3+A*V)k&iad6kW^YeOF zAFSpwGIiRCa19nkLd-wxUaXhHKX@>Sd?ovmC(s&HI=k0U5;Xv z?=w-WqrR+p;Ym zDZ%o6oV8ffvzj$}r*Aa??^H|H^{eoXntdXAw&3krqm^d@`H;A6W$`A=Zivd#;$0Xe zPi&0<>I(#4h-at)R`w*)o zXf5zzfLKm+rAzr|AP$g_yimd)bpJc&BXwXPm{NUO7^aYbpupYcQaBxTXTZlG+Ib6n zfU^eFH?FQ$zXprLt+Oz;kK|aa;#3GQH^rv=RES}-TnIIT?e|4$20QORtIgY6e?sVw zvW;^|#O?p(Ii`p^&SvDkRq)_GA)c z*ju?!i$dI{OiNeXrb0^?I`7Cj16T4IP-6**qlJ;83-Lg0^n!y8hnSEz(LK=g?T}(< z`cB9uGzwNb1~+iDF2XU~dmpFv2T?H}ku84-6|-l>w4DM zY^d7avlv3C6H8<@{x{h8tJ6}=vWlz@6Xpzp%9vsdx;d+NvPhON`)YwzFsW)hEf5rv zy#F^^KqO{LZ!F@f4C_#da0 z|CihRZzJlz7^)8UES>`Y(JLV}oSjhPAHB9`h@|>$D~3jzAAdlNoqjTpzM#~>mM7XN zHjjTPaISjCdB`!SM!T$^=Ml@bD&w+m&Q1}!Cx9aa7l8K$GAoRHLGjQH4G89Z4xJYT z(keou=v3CteTZz2%!!EDm+X5ObW;-1YQpeJr@*P@%T^=?LL6isSEYs@TR9iuTPtV* z3wf#c20X6EE^bx*1g)HQ%9Iv1=0B=qP66**COVNOU1H&rR4p;zym;CiiZy<(4gO=FZU}`cSYs{_b6`+M zXnT+}n6{KnUkOa;(&JlFYFw3yhqc7iw#NihX8VZG7-PD3+Oefz69rs=tXE&_9~Iz`iEXyI3%u;r=T;b508q}fLo=vv zMC={{5;%3f0ysFxfym5(M8tSD4~uiX*Pv8#`KC|@)rntQP2yLD=?NwI*`E)j*9`xD zWSrkGs(q0Lhnv~d5ld5#rKv2ilZ$b2Xh<&U%%${9@Gi0qEJ~nqDerwHR#Mg2Z;oK! zsZ%pQa1F{=9OMC~h(ptD(cA8Ube@U+NKtBUujA1+(oW`k`P*>6B*hZU3@5?avGZ@| zxy>YS3{^7~hK!IjG}sHl7uKecX5buBb&?#CQE(2~D2W-EAKb~&p%AFlCVLxlg?6H_ zuJBFo8~3-I$Q&@h9xNXyBN-y{m@>LamJpeeYcWD9qyiYogj2u#CSdW1YLk1ErsGiX zglZUg85na#cd`_P0|byqI2pO%naw@s#hE$KJc@a&H+)}jSio0R%EH|^k`gh&36!fA z&ekF}Km zH|DGQ!-9F}#)ccHE)SfC1Bh|gBM1M^#5VCkAi{rTVsI~iXB+sRJZwnU#BEU+62wsb zAbo5y9>Lc8gq8d=9R^m5rM{$jRlV%IESF~Pw(Q~9L&rf96G`!ewSw3Ky72w!}LXBui~!fZ9Vm%7f1UFFic=)SZKte|U> zT>O3|(OmABKcC`JoIrtJ@HhvUdsNn!f-yZqtQ2nud_|Nx*^5{yi7r02jb$sNdgf!>I`XBU)4{4Bqxuf^|RvH~s#0lpK z>6+Az`itPzLz~zPW7Zqm9n9*p4xxaOzH#FGMt*x%yD*CyqxdCqtHSh?CSRebjVs}3 zTLc3RQ(%SZVz4+FmWu;o-NF7BE#23vectyuPhPdUM9*Gs_+y_dT-F-B9WW}xE0x}A zqEj*W;sBJP>C`MO-#Qs&N)^r|78z&?)FR)ylX=qh{ZuYBepRSh{_B->I28vD?+w_O zFfieMsX65YuY_#N zr7liU`Mqj|>T#XwQeO2degxi|m9s*%Pb6hd!7zY)NVJlseao5S@j`knrn0t(Y(Uu@ zWtyX-5z&!@Uedy;jfp%Y*FN6cKlyF{L(U6aA~AdP?<4Qb1vsy>*i+w82v=0H*c+k` zpCZDyH4v{~8112x@l`##CIzY=s;(B(A~+d}Cghm2HPO=-1-f%oguW_7%+XUtEZnG& zqS*rDrp0Wr5ie_6U2k@&`K9c`dMLLhjW&(-)Uka1TiXI^qFa5q&+uXO4iV1jqS=Iw zGS6RnuD*u9PG3JebE`Lxz{{`5xsV7*T#zZey1gCOwpk&KXu6t{<_ZJ6Uh#ATE4;27 z(@Wy+w#$~A@5rEH9`*WS8coLKx_fi@Z9gHi8d{h9hEEn>$oSl>wXKacmEBh(Rl(Ba z$J48MHRM^CU-gLAh(5TF628792K`CHfbHjH?9qQyb_xdsg8VmS?JYcQthKy69bE1H zVXl{ss@sMzxSetvU?iyDrf(|dRE8KW;!pdKV%)#Ab|c|=o!ghk`z?u9Lv8o&qDaIj z7hI>%kuKtP9zGTO^iC^#48Oj?dH00zN)a2^aeBMF8KyQp9 zda4?H?_li`LEG4{fvVXX=_HQOrOQjYw1S5($f-=~*1~%)1?smb3L6yOREiUFF`ZN= z*+@x$Y@S;3la_-?r?v!kbe$7Mt)r~12m4DitP;ASatv7}2hFb1mnVdB&HH0jC5OZ$ck2!!sb9x&7Wecz z6XC653_sU)l~QY65<1QeP0JIWiY64%$aD^vG3Gc{Hzbtosdg9^2Kim2@f`*EaZQss z5-5+_Z4ZoS{r;|PvsJ0uX&hX$rI|e3-QqBtt)0C!@S?$hA1JnOHLDcfg9XUe$vss+ z6EiF?8N73@=}>U!2+;ShP@8mam5OYpD}!{ z>mPgFF)IA&?$_yx+Tie=Wd`5FoQlSIFP ztwDY=U-f%i%m!t(Ii}u;3SfPWjUb?ujS!9A5XH++tOZ_Oh-FX}upvdm)68|LLdl+E z5ov4VB1&Z^ziolA1ZcGiIBi4l+({HexZ;W;k!kSV3vA<1L}iP%td;z`o}@?dT4`J z3BTy(yVhk)Ceb{^7uYyjOd$exT1i;IIuWI<54IDE`EHNN#ToApp%T&UR-BtPm}+q1 zC!bnou*aHc@%6;dJIvM-LF&b1e37|9U&dWJ$Css7ycbFl@fvaQegSAI?4+VG@6Cf(8P4)&;UNqX8!BUm2;wj#O`n z34hVmFHI#j#4yZ3Jy{u_d)d_}i-@V;jF5DNo5s zO#%tolvix$8OzV;*5ma2;DCY_rB29L3ex)mHP<*wZPiEa+T&Ht9vZyE#1Y$9Chqz}A+<$!9%Gk269U}j`wP7Pa@h~9=u!+F`v-D?t zjRyiD7qFp7fCo(bpZXREMD@Q{?8TU$TD6I+JAcjItWDapUuB#<7~)%UMVyinZ8tY!61Lag4REeX*}^b#$4kIqIXIuR{u z^v>sbzVCb9^S=@;OXdMYwqZ1`^d}Qt^_*b@L7WTohkAbdtWn(h&({`Ih$XN_LHJf^~OtG z$t7wn?5Y~1HuP}MRYQJMQ?FRvD!a4$Wc4g{%Qfv#E2Tj=y)ZUX934JnHJkx^mTr;MCv-8!+1%8Pfi$SzlgxfncOb zTD+%8!n7h{!M@Zsw&q&h=AHIOq^@GoS6BosS~7&qbj*_~+xAy9v^-*mqMNf}YW5sL zk8gthqD>-*I>5T`-}21)DHBomC%uS>5N@eoQ8+BSZr4TYZ?_Hm+JAPZ-pmTgWtF#? zX41Th3x)GIhVe#e>Y~G2l*waS;*v)~)QIYl^@?iUmTHIQ$m#%8ZoL$y=}mx-%6K&m z7e0-$oG%A&82d(4X`eIcbx?)Mm#Sb^6C;(`bru~0RKKFcr&9OJJtyF`R34-b4hhf^Yr+Eu(B*U zyzt`^CTgtnByVc#L6-Q3i&?sYeM@t1#d;|Ad^8 zdRm);e*US=F-%Y_2xdK}x=Q-~o0pRkP)&p9vltiE%v$qYcI3MgqTb8g$a%XlArTI8 zjEpzjv?Qo}b+Z{LZs2!O%kz>+xAxJtFK|A=i2n^;oeNdXM!RIPNj%z#`_>%7em1cwf&5g zYLSkD5k=}CH8~xh98%xJ>kGBf4N(56x9VdY!jT-WGIwdudCtYL2$ZOJWggDL2W_Zg z$BUi0JiQOpQHHwC;(wY7jVL&sLIukXYW;K--^q+?J}OSOuCYx0{kiK~LH|07hdgiP zs%-VKLfq%cj3TURo2kxMt<(AD^>I<)W74IDO0c13Yc6ABh1!7#C97%Z%JVOD#%Bvf zd}w+75dy@w(5jpd^5Q7&M1m}xr`Xw$fO9K0madtm0K(I%%AE56WsA55BSRMJf0k|~i-HR@J}hYZSma0SCQ}>8qb9-sIFq+#HU-6RTNOIaP${ZL4Drw= zcZfD7aH*%{r9)8{Qg2C9C;g0!4@|hG@Few#xtXs|=(w*#-+LSSr@5HbSW~i9{Lu|G zeO!cSAwd~f3u)Svq}MIA)`` znarYrFXF!i*^`_NkMPI?`%_Mkj=bhQ!Ou&cN$BvN`H0G?@i9!Kz~@f3FnOH|JGfmZ z$KfK-@S9B{pCVAo7+Z-O{+2i4j;EoPX(%&FF3qd$Xl<(iiyp8IXz650JvZI2wPMyh z=l=MJ^7jYI-nkH-8+h^q85eVJ7|>=VWsVO?MS$5eSOioos34YIOF(auKI;gaoCZ%{f}LFYA4 z_Bn6c^een3G-&&z<4s4FvrcoB^!a5Kf7fTZ=Zo>3Y+}@56XaPHGy!-W`l(K#%604a zq%R5ndZksFv{3}aXvUmmHexKS;Ain&09KS~Otv6(*P{6J-tdtLaae(3s$c`$Dnsn@ zv_9dzsCIvi<+$Bi;~qZ9VflR$Ua|{MtLuGf6X`Tvjo0l4g()BIrE3w1%F|U&=O3-d z_*7^UNA` zf~mRZCFGN1Lwn(ip-v=iCZ{|kR7lLFE}zKO8LlJgl>N?h;>b5W!>1Ac+$kbO;fH3B z9zVNBWy0=LS)P~dZ>Gg({+L9yXvuZrOFn3lZ_3wC%JR+6$q}5{9DSpVNKD_`5txp1 z7K+=weilT;j#z@8FnTEj@7eNviYobox|h;IMH^Fo-+}(l1lE5u@nsu&S=HHnArMA8{L>x~}W1SITVcp)|#Y?xhA5x|rW|hg6P+h|Wog)dPHU0_wYLhLFjv+m5 zPc82`$0nU$vRQh>H`Oj1`I1ER)S`^+CfIm9zP!Hi@x6I-dUm+Kw%KEPYkXDbAj(K5 zfO_VO>bp5Ey)Ja#cdR-*JKBl4-Ay-5Pj9_#a`3qtwwW`QUbXA}xdmx-Mo*YlZkDC& zCKeBvit8t8;g5PNNztrWq}8p)BGT$CvY}iT-HKA~E9qRLU#M_78~gwd{847ik+Bzo zAWoTya3QLUt)%j!Lx*eb3F-Hit{Xlt zvJ8QX-^2^p8prg=z$XqM%GPyHgAG?@x*R7K^p#jPJ_t}m(2k>E5y09;5EUsZEJiN4 zq6pHF9#+rCxcAHFgWda8^Gn?OAt0lu?IYt8sntyMXNV>PQ9JC93-YTzos?kC4ewsn ztd~k|0!iUI>LA4C7r&fM?Nk-pv4JIZ6Sq-w2;J(q9gpO(o?c-guQ&KlF>k0G6c)}5 z$|(xtfSP3uvE2zPRwsNuH*&|U>Q2_`n) zIIrLL>+z!Qs^F);fs=UI`*iPs3GamSq@A`1vLb?)&~Ch!(7zA*|3+1YV#~s=@oxDT9DDYJ##& zA@E1wv|7v&B9l~vo*@I^DWRhXp8eWV}gPbt@%fyur zMC?RBp?0)MqlgOg*~qi(iAv9F%i@vyd)?~2h!3f6N$gLKQPN@qX5|mm%X$c>!R+TM zNZ<2e#kBBW@0bI86Uvm9h7KkIy^Ij*EE%k%cp0pZg(q<}-1>F%F>d{Lw!RC^jP;}} z|IhQIg*wbwfDlDU-)Gg_6TIoGGv@d-B4^(18R9ogFZ#TM z3Dezf$AlSd-)6$Jw*#3leeG}~>cdB+rzR<)2-$5RzHr5a%u#qDyg2gK92TW0T*X@X zAT*{Zw_)gjDdXlE^`q=kq1ZE;X~yq6;1VpHJ5)aE=t6cUtw&MRH8;F14}SuR^qQ+W zSb{;w5OzXmTc1Eyo4rYrE^B0npRfcn#42o(4DkzALxxy~okSn{IIaBmK4e7RM!Z9a z0(Q15DJQYM`N{;N+ChWM{iN z69&_cVRC&f@hR7$5??&(z3;vF32PO(MB5oo+U6V4@rt`>jHP^BtQM;3SQOK0 zoyOBgoWz|S40GPrlam}+EL*IgK+6;dRLfK=`JMT_J)IUSR-IP8T%A-aI?Fr}AWX7QNQ;!EbYX>boa?8AC zjH+(*HGd>A8H@$bgi5toCY0=_ktiLpRqeX#i#9ZjM|oM9SlZVzeBaI7D-AhaSy~!{1Q?4a0c6CsOhdn(TwU^V~G`9Ix^guu!E_b9pQGiVXR9sZc~>TL|vI7)MM=IEF3s{TAtmCr7^q-diCl4&I1&K z?TEep`vvC4j`7qdiy+!lsG0RTQ+WOgB(GHtY^pWpn45EVXV%=3O za#IFDL?l;=eT!ziP?N8^-sz(z$^#Oc$Et%?8^sHK2Zg1@3(?;{Jz~Ms$3dUa&+g>W zReI{a>8(j|^Y^%%c-yE_`P?mKZGgix+pWrA$8qetwCcgK+0KUs<;q#NOP)Ry-6b@x znd;mknwz3JdNay;je`w%d~6`hp=Y*{;=X zx8EtqAJwCpolkZewy;`W0clV#GP0UN#w%5#Qbm zo>DK#;wlOqE6@}PC6X0H6rNP%tA!Y@h!|6@{$d4mw2}Dq`$$H=RJ27s{M5!4gjc0) z3v}(T;0la8H0X}#b^yGVZx4=Pv!Dma;={SrPQnap3yhbd&+m-OUhW5Tn7`dG^u$eAr5rQSf5;9uHU z+@!pb8xq|SIti(~Kp|9?kVS;~Pc?tJ%gPTF*sReGGV#Wp6dlaDBSwJylMP!Ydz2Hx9U{2@sJ%G^-`@vkUDq~ z|5l;TJ27wGE#wRCN9iz~cCxP8*g1!GP9Bx7yNf9`vvV(knmE?y_R_soWNx+HkBBB! zK3j!FQd0Y`L6n=*v!L~1`P?q)$ab60W+638!%Y&;2Hq&tSNwr%;7+PJy<|l`e-L zujfS|&JGhNC;-*OT;hVFJ?cm6sI=ODQ*W&6v1=#TU%GKQfZ5G@j3eGf#%3)2SiN?A zW&?JwnnB6I~cKms*sKJ`+wg@f1X8rPc;qcg6dOt9@o@a@NOTq zOC0P*u+mUurf15Q49d@A@M@Lx$h78d=#>59_F z>w$TJ|8mk;Sk(Wf@SZ(hh@Ce65kEQbpX-2wg+=jiN|v1QulS|0PvbxFgYGr+{k8UA DX}E)! diff --git a/biodb/apps/uploader/tests/test_io.py b/biodb/apps/uploader/tests/test_io.py index b18ad6a..6016da5 100644 --- a/biodb/apps/uploader/tests/test_io.py +++ b/biodb/apps/uploader/tests/test_io.py @@ -10,12 +10,12 @@ from uploader.tests.conftest import DATA_PATH -SPECTRAL_FILE_PATH = (DATA_PATH/"sample") +ARRAY_FILE_PATH = (DATA_PATH/"sample") @pytest.fixture(scope="module") def json_data(): - filename = SPECTRAL_FILE_PATH.with_suffix(uploader.io.FileFormats.JSONL) + filename = ARRAY_FILE_PATH.with_suffix(uploader.io.FileFormats.JSONL) with open(filename, mode="r") as fp: return json.load(fp) @@ -23,7 +23,7 @@ def json_data(): class TestReadSingleRowArrayDataTable: @pytest.mark.parametrize("ext", uploader.io.FileFormats.list()) def test_read(self, json_data, ext): - filename = SPECTRAL_FILE_PATH.with_suffix(ext) + filename = ARRAY_FILE_PATH.with_suffix(ext) data = uploader.io.read_single_row_array_data_table(filename) assert str(data.patient_id) == json_data["patient_id"] @@ -36,7 +36,7 @@ def test_multiple_row_exception(self, ext): class TestArrayDataFromJson: def test_read(self, json_data): - filename = SPECTRAL_FILE_PATH.with_suffix(uploader.io.FileFormats.JSONL) + filename = ARRAY_FILE_PATH.with_suffix(uploader.io.FileFormats.JSONL) assert uploader.io.array_data_from_json(filename) == uploader.io.ArrayData(**json_data) def test_array_data_from_json_key_validation(self): diff --git a/biodb/apps/uploader/tests/test_management_commands.py b/biodb/apps/uploader/tests/test_management_commands.py index 25a3277..0a6643b 100644 --- a/biodb/apps/uploader/tests/test_management_commands.py +++ b/biodb/apps/uploader/tests/test_management_commands.py @@ -97,7 +97,7 @@ def delete_objs(model): @pytest.mark.django_db(databases=["default", "bsr"]) class TestGetColumnNames: - n_non_observables = 28 # 39 including instrument fields. + n_non_observables = 22 @pytest.fixture def more_observables(self, centers): @@ -140,7 +140,7 @@ def test_all(self, observables): out.seek(0) assert len(out.readlines()) == Observable.objects.count() + self.n_non_observables - @pytest.mark.parametrize("center_filter", ("jhu", + @pytest.mark.parametrize("center_filter", ("JHU", "imperial college london", "oxford university", "d2160c33-0bbc-4605-a2ce-7e83296e7c84", @@ -158,7 +158,7 @@ def test_center_filter(self, observables, more_observables, center_filter): out.seek(0) assert len(out.readlines()) == queryset.count() - @pytest.mark.parametrize(("center_filter", "category_filter"), (("jhu", "bloodwork"), + @pytest.mark.parametrize(("center_filter", "category_filter"), (("JHU", "bloodwork"), ("imperial college london", "comorbidity"), ("oxford university", "drug"), ("d2160c33-0bbc-4605-a2ce-7e83296e7c84", "bloodwork"))) diff --git a/biodb/apps/uploader/tests/test_models.py b/biodb/apps/uploader/tests/test_models.py index 1b84433..42c19ee 100644 --- a/biodb/apps/uploader/tests/test_models.py +++ b/biodb/apps/uploader/tests/test_models.py @@ -53,7 +53,7 @@ def test_editable_patient_id(self, center): assert Patient.objects.get(pk=patient_id) def test_center_validation(self, centers): - center = UploaderCenter.objects.get(name="jhu") + center = UploaderCenter.objects.get(name="JHU") assert UserCenter.objects.filter(pk=center.pk).exists() assert UploaderCenter.objects.filter(pk=center.pk).exists() @@ -66,10 +66,10 @@ def test_center_validation(self, centers): patient_id = uuid4() with pytest.raises(ValueError, match="Cannot assign"): patient = Patient(patient_id=patient_id, - center=UserCenter.objects.get(name="jhu")) + center=UserCenter.objects.get(name="JHU")) def test_unique_cid_center_id(self, centers): - center = UploaderCenter.objects.get(name="jhu") + center = UploaderCenter.objects.get(name="JHU") cid = uuid4() Patient.objects.create(patient_id=uuid4(), center=center, @@ -94,7 +94,7 @@ def test_pi_cid_validation(self, centers): id = uuid4() patient = Patient(patient_id=id, patient_cid=id, - center=Center.objects.get(name="jhu")) + center=Center.objects.get(name="JHU")) with pytest.raises(ValidationError, match="Patient ID and patient CID cannot be the same"): patient.full_clean() @@ -274,7 +274,7 @@ def test_observable_value_bool_cast(self, db, observables, visits, value): assert observation.observable_value is value def test_center_validation(self, centers): - center = Center.objects.get(name="jhu") + center = Center.objects.get(name="JHU") patient = Patient.objects.create(center=center) visit = Visit.objects.create(patient=patient) @@ -369,19 +369,19 @@ def test_all_files_deleted_upon_transaction_failure(self): ... @pytest.mark.parametrize("ext", uploader.io.FileFormats.list()) - def test_clean(self, centers, instruments, ext, bio_sample_types, spectra_measurement_types): + def test_clean(self, centers, instruments, ext, bio_sample_types, array_measurement_types): data_file = (DATA_PATH/"sample").with_suffix(ext) data = uploader.io.read_array_data(data_file) patient = Patient.objects.create(patient_id=data.patient_id, - center=Center.objects.get(name="jhu")) + center=Center.objects.get(name="JHU")) visit = patient.visit.create() bio_sample = visit.bio_sample.create(sample_type=BioSampleType.objects.get(name="pharyngeal swab")) array_data = ArrayData(instrument=Instrument.objects.get(pk="4205d8ac-90c1-4529-90b2-6751f665c403"), bio_sample=bio_sample, data=ContentFile(data_file.read_bytes(), name=data_file), - measurement_type=ArrayMeasurementType.objects.get(name="atr-ftir")) + measurement_type=ArrayMeasurementType.objects.get(name="magic")) array_data.full_clean() array_data.save() @@ -408,7 +408,7 @@ def test_upload_without_error(self, file_ext, center, bio_sample_types, - spectra_measurement_types): + array_measurement_types): meta_data_path = (DATA_PATH/"meta_data").with_suffix(file_ext) array_file_path = (DATA_PATH / "array_data").with_suffix(file_ext) with meta_data_path.open(mode="rb") as meta_data, array_file_path.open(mode="rb") as array_data: @@ -431,7 +431,7 @@ def test_mock_data_from_files_fixture(self, mock_data_from_files): def test_center(self, mock_data_from_files): n_patients = Patient.objects.count() assert n_patients == 10 - center = Center.objects.get(name="jhu") + center = Center.objects.get(name="JHU") assert n_patients == Patient.objects.filter(center=center).count() assert not Patient.objects.filter(center=None) @@ -448,7 +448,7 @@ def test_number_observations(self, instruments, center, bio_sample_types, - spectra_measurement_types): + array_measurement_types): """ The total number of observations := N_patients * N_observables. """ assert Patient.objects.count() == 0 # Assert empty. @@ -565,7 +565,7 @@ def test_bad_ext(self, instruments, center, bio_sample_types, - spectra_measurement_types): + array_measurement_types): meta_data_path = (DATA_PATH/"meta_data").with_suffix(UploadedFile.FileFormats.CSV) array_file_path = (DATA_PATH / "array_data").with_suffix(UploadedFile.FileFormats.CSV) @@ -587,7 +587,7 @@ def test_bad_ext(self, @pytest.mark.django_db(databases=["default", "bsr"]) def test_get_center(centers, mock_data_from_files): - center = Center.objects.get(name="jhu") + center = Center.objects.get(name="JHU") assert get_center(center) is center from user.models import Center as UserCenter diff --git a/biodb/apps/uploader/tests/test_uploader_admin.py b/biodb/apps/uploader/tests/test_uploader_admin.py index 2b6a2a3..e9de471 100644 --- a/biodb/apps/uploader/tests/test_uploader_admin.py +++ b/biodb/apps/uploader/tests/test_uploader_admin.py @@ -53,7 +53,7 @@ def staffuser(centers): user = User.objects.create(username="staff", email="staff@jhu.edu", password="secret", - center=UserCenter.objects.get(name="jhu"), + center=UserCenter.objects.get(name="JHU"), is_staff=True, is_superuser=False) return user @@ -64,7 +64,7 @@ def superuser(centers): return User.objects.create(username="admin", email="admin@jhu.edu", password="secret", - center=UserCenter.objects.get(name="jhu"), + center=UserCenter.objects.get(name="JHU"), is_staff=True, is_superuser=True) @@ -194,7 +194,7 @@ def test_perm_without_user_center(self, django_request, instrument): @pytest.mark.django_db(databases=["default", "bsr"]) class TestUploadedFile: def test_non_form_field_validation(self, mock_data_from_files): - # Note: ``mock_data_from_files`` uses Center(name="jhu")``, so create a new user of a different center. + # Note: ``mock_data_from_files`` uses Center(name="JHU")``, so create a new user of a different center. user = User.objects.create(username="staff2", email="staff2@jhu.edu", password="secret", diff --git a/biodb/apps/user/fixtures/centers.json b/biodb/apps/user/fixtures/centers.json index 075fa0d..cd0e433 100644 --- a/biodb/apps/user/fixtures/centers.json +++ b/biodb/apps/user/fixtures/centers.json @@ -19,8 +19,8 @@ "model": "user.Center", "pk": "16721944-ff91-4adf-8fb3-323b99aba801", "fields": { - "name": "jhu", - "country": "UK" + "name": "JHU", + "country": "USA" } }, { @@ -43,8 +43,8 @@ "model": "uploader.Center", "pk": "16721944-ff91-4adf-8fb3-323b99aba801", "fields": { - "name": "jhu", - "country": "UK" + "name": "JHU", + "country": "USA" } } ] diff --git a/biodb/apps/user/tests/test_center.py b/biodb/apps/user/tests/test_center.py index fd02943..e479999 100644 --- a/biodb/apps/user/tests/test_center.py +++ b/biodb/apps/user/tests/test_center.py @@ -9,8 +9,8 @@ @pytest.mark.django_db(databases=["default", "bsr"]) class TestCenters: def test_centers_fixture(self, centers): - jhu_from_user_table = UserCenter.objects.get(name="jhu") - jhu_from_uploader_table = UploaderCenter.objects.get(name="jhu") + jhu_from_user_table = UserCenter.objects.get(name="JHU") + jhu_from_uploader_table = UploaderCenter.objects.get(name="JHU") assert jhu_from_user_table.pk == jhu_from_uploader_table.pk assert jhu_from_user_table.name == jhu_from_uploader_table.name From e83fc8c661abf9618708db1ad10699b26d5d6924 Mon Sep 17 00:00:00 2001 From: James Noss Date: Thu, 30 May 2024 09:24:56 -0400 Subject: [PATCH 4/5] Correct scripts Signed-off-by: James Noss --- .dockerignore | 2 +- .gitignore | 2 +- docker-compose.yml | 10 +++++----- nginx/nginx.conf | 4 ++-- scripts/dev/rebuild_postgres.sh | 4 ++-- scripts/dev/rebuild_sqlite.sh | 4 ++-- scripts/prd/deploy.sh | 2 +- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.dockerignore b/.dockerignore index 217ef67..c1be92c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -13,7 +13,7 @@ coverage.* __pycache__ *.sqlite3 db -spectral_data +array_data raw_data datasets log diff --git a/.gitignore b/.gitignore index 918e917..5273a46 100644 --- a/.gitignore +++ b/.gitignore @@ -170,7 +170,7 @@ cython_debug/ # Project stuff raw_data/ -spectral_data/ +array_data/ datasets/ log/ run/ diff --git a/docker-compose.yml b/docker-compose.yml index 955b3d0..5b25719 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,7 @@ services: restart: "no" volumes: - db-data:/app/db - - spectaldata-files:/app/spectral_data + - arraydata-files:/app/array_data - bulk-upload-files:/app/raw_data - spectaldata-files:/app/datasets environment: @@ -28,7 +28,7 @@ services: && python manage.py migrate && python manage.py migrate --database=bsr && python manage.py loaddata centers queries - && python manage.py loaddata --database=bsr centers observables instruments qcannotators biosampletypes spectrameasurementtypes + && python manage.py loaddata --database=bsr centers observables instruments qcannotators biosampletypes arraymeasurementtypes && python manage.py update_sql_views flat_view && python manage.py crontab add && python manage.py prune_files @@ -51,7 +51,7 @@ services: - 8000 volumes: - db-data:/app/db - - spectaldata-files:/app/spectral_data + - arraydata-files:/app/array_data - bulk-upload-files:/app/raw_data - dataset-catalog-files:/app/datasets - static_files:/app/static @@ -87,7 +87,7 @@ services: - "443:443" volumes: - static_files:/static - - spectaldata-files:/spectral_data + - arraydata-files:/array_data - bulk-upload-files:/raw_data - dataset-catalog-files:/datasets depends_on: @@ -105,7 +105,7 @@ secrets: volumes: db-data: - spectaldata-files: + arraydata-files: bulk-upload-files: dataset-catalog-files: static_files: diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 5b80a45..80fda92 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -44,7 +44,7 @@ server { alias /raw_data/; } - location /spectral_data/ { - alias /spectral_data/; + location /array_data/ { + alias /array_data/; } } diff --git a/scripts/dev/rebuild_postgres.sh b/scripts/dev/rebuild_postgres.sh index 7294b53..2522d08 100755 --- a/scripts/dev/rebuild_postgres.sh +++ b/scripts/dev/rebuild_postgres.sh @@ -1,5 +1,5 @@ # Delete uploaded data files. -rm spectra_data/* +rm array_data/* rm raw_data/* rm datasets/* @@ -54,7 +54,7 @@ python manage.py migrate --database=bsr # Load initial data fixtures. python manage.py loaddata centers queries -python manage.py loaddata --database=bsr centers observables instruments qcannotators biosampletypes spectrameasurementtypes +python manage.py loaddata --database=bsr centers observables instruments qcannotators biosampletypes arraymeasurementtypes # Update SQL views. python manage.py update_sql_views flat_view diff --git a/scripts/dev/rebuild_sqlite.sh b/scripts/dev/rebuild_sqlite.sh index ca8cb2e..994fa03 100755 --- a/scripts/dev/rebuild_sqlite.sh +++ b/scripts/dev/rebuild_sqlite.sh @@ -1,5 +1,5 @@ # Delete uploaded data files. -rm spectra_data/* +rm array_data/* rm raw_data/* rm datasets/* @@ -25,7 +25,7 @@ python manage.py migrate --database=bsr # Load initial data fixtures. python manage.py loaddata centers queries -python manage.py loaddata --database=bsr centers observables instruments qcannotators biosampletypes spectrameasurementtypes +python manage.py loaddata --database=bsr centers observables instruments qcannotators biosampletypes arraymeasurementtypes # Update SQL views. python manage.py update_sql_views flat_view diff --git a/scripts/prd/deploy.sh b/scripts/prd/deploy.sh index 43c8b46..57b3c5e 100755 --- a/scripts/prd/deploy.sh +++ b/scripts/prd/deploy.sh @@ -17,7 +17,7 @@ pipenv run python3 manage.py migrate --database=bsr # Don't do this on deployment so as not to clobber any live alterations. Instead, call these manually from the ec2 # instance connected to the RDS instance. #pipenv run python3 manage.py loaddata centers queries -#pipenv run python3 manage.py loaddata --database=bsr centers observables instruments qcannotators biosampletypes spectrameasurementtypes +#pipenv run python3 manage.py loaddata --database=bsr centers observables instruments qcannotators biosampletypes arraymeasurementtypes # Update SQL views. pipenv run python3 manage.py update_sql_views flat_view From 24fabded532dac1ab09f188a636eff4a9709e329 Mon Sep 17 00:00:00 2001 From: James Noss Date: Thu, 30 May 2024 09:32:17 -0400 Subject: [PATCH 5/5] Minimal corrections to docs to get them to build Signed-off-by: James Noss --- docs/source/api_data.rst | 6 +++--- docs/source/api_explorer.rst | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/source/api_data.rst b/docs/source/api_data.rst index 45e12fc..60561ea 100644 --- a/docs/source/api_data.rst +++ b/docs/source/api_data.rst @@ -28,15 +28,15 @@ Observations uploader.models.Observable uploader.models.Observation -SpectralData +ArrayData ------------ .. autosummary:: :toctree: generated uploader.models.Instrument - uploader.models.SpectraMeasurementType - uploader.models.SpectralData + uploader.models.ArrayMeasurementType + uploader.models.ArrayData Bulk Data Uploads diff --git a/docs/source/api_explorer.rst b/docs/source/api_explorer.rst index 8093e51..eb393bc 100644 --- a/docs/source/api_explorer.rst +++ b/docs/source/api_explorer.rst @@ -20,7 +20,7 @@ Data Exporters CSVExporter ExcelExporter JSONExporter - ZipSpectralDataMixin + ZipArrayDataMixin Charts ------