Merge pull request #9 from ssec-jhu/adapt-models

Adapt models

Author: jamienoss

.dockerignore

@@ -13,7 +13,7 @@ coverage.*
 __pycache__
 *.sqlite3
 db
-spectral_data
+array_data
 raw_data
 datasets
 log

.gitignore

@@ -170,7 +170,7 @@ cython_debug/
 
 # Project stuff
 raw_data/
-spectral_data/
+array_data/
 datasets/
 log/
 run/

biodb/apps/catalog/admin.py

@@ -27,7 +27,7 @@ def has_delete_permission(self, request, obj=None):
 @admin.register(Dataset)
 class DatasetAdmin(AuthMixin, admin.ModelAdmin):
     search_fields = ["created_at", "name", "version", "query__name"]
-    list_display = ["name", "version", "file", "created_at", "size", "n_rows", "n_spectral_data_files"]
+    list_display = ["name", "version", "file", "created_at", "size", "n_rows", "n_array_data_files"]
     date_hierarchy = "created_at"
     ordering = ("-updated_at",)
     list_filter = ("name",)
@@ -40,8 +40,8 @@ class DatasetAdmin(AuthMixin, admin.ModelAdmin):
                        "updated_at",
                        "id",
                        "n_rows",
-                       "n_spectral_data_files",
-                       "spectral_data_filenames",
+                       "n_array_data_files",
+                       "array_data_filenames",
                        "data_sha256"]
 
     fieldsets = [
@@ -73,14 +73,14 @@ class DatasetAdmin(AuthMixin, admin.ModelAdmin):
                            "app_version",
                            "id",
                            "n_rows",
-                           "n_spectral_data_files"],
+                           "n_array_data_files"],
             }
         ),
         (
-            "Spectral Data Filenames",
+            "Array Data Filenames",
             {
                 "classes": ["collapse"],
-                "fields": ["spectral_data_filenames"],
+                "fields": ["array_data_filenames"],
             }
         ),
     ]
@@ -91,9 +91,9 @@ def size(self, obj):
             return f"{int(obj.file.size / 1e6)} MB"
 
     @admin.display
-    def n_spectral_data_files(self, obj):
-        if obj.spectral_data_filenames:
-            return len(obj.spectral_data_filenames)
+    def n_array_data_files(self, obj):
+        if obj.array_data_filenames:
+            return len(obj.array_data_filenames)
         return 0
 
 

biodb/apps/catalog/migrations/0001_initial.py

@@ -30,8 +30,8 @@ class Migration(migrations.Migration):
                 ('app_version', models.CharField(blank=True, default=catalog.models.get_app_version, editable=False, help_text='App version used to create data product', max_length=32)),
                 ('sha256', models.CharField(blank=True, editable=False, help_text='Checksum of downloadable file', max_length=64, validators=[django.core.validators.MinLengthValidator(64)], verbose_name='SHA-256')),
                 ('n_rows', models.IntegerField(blank=True, editable=False, help_text='Number of data rows')),
-                ('data_sha256', models.CharField(blank=True, editable=False, help_text='Checksum of data table (not including any spectral data files).', max_length=64, validators=[django.core.validators.MinLengthValidator(64)], verbose_name='Data SHA-256')),
-                ('spectral_data_filenames', models.JSONField(blank=True, default=catalog.models.empty_list, editable=False, encoder=catalog.models.CustomDjangoJsonEncoder, help_text='List of spectral data filenames')),
+                ('data_sha256', models.CharField(blank=True, editable=False, help_text='Checksum of data table (not including any array data files).', max_length=64, validators=[django.core.validators.MinLengthValidator(64)], verbose_name='Data SHA-256')),
+                ('array_data_filenames', models.JSONField(blank=True, default=catalog.models.empty_list, editable=False, encoder=catalog.models.CustomDjangoJsonEncoder, help_text='List of array data filenames')),
                 ('query', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='dataset', to='explorer.query')),
             ],
             options={

biodb/apps/catalog/models.py

@@ -46,7 +46,7 @@ class Dataset(DatedModel):
             sha256 (:obj:`django.models.CharField`): The SHA-256 checksum of the entire zip file.
             n_rows (:obj:`django.models.IntegerField`): The number of rows in the zipped data file. Depending on the query, this could be the total number of patients or something else.
             data_sha256 (:obj:`django.models.IntegerField`): The SHA-256 checksum of the data file archived within the zip file.
-            spectral_data_filenames (:obj:`django.models.JSONField`): A list of all the file names for all individual spectral data files zipped within the downloadable zip file.
+            array_data_filenames (:obj:`django.models.JSONField`): A list of all the file names for all individual array data files zipped within the downloadable zip file.
     """
 
     class Meta:
@@ -99,13 +99,13 @@ class Meta:
                                    null=False,
                                    blank=True,
                                    verbose_name="Data SHA-256",
-                                   help_text="Checksum of data table (not including any spectral data files).",
+                                   help_text="Checksum of data table (not including any array data files).",
                                    validators=[MinLengthValidator(64)])
-    spectral_data_filenames = models.JSONField(null=False,
+    array_data_filenames = models.JSONField(null=False,
                                                default=empty_list,
                                                blank=True,
                                                editable=False,
-                                               help_text="List of spectral data filenames",
+                                               help_text="List of array data filenames",
                                                encoder=CustomDjangoJsonEncoder)
 
     def __str__(self):
@@ -124,7 +124,7 @@ def clean(self, *args, **kwargs):
         if not self.file:
             # Create file from query.
             file, info = self.execute_query()
-            filename, n_rows, data_sha256, spectral_data_filenames = info
+            filename, n_rows, data_sha256, array_data_filenames = info
 
             if not n_rows:
                 raise ValidationError(_("Query returned no data."))
@@ -133,7 +133,7 @@ def clean(self, *args, **kwargs):
             self._filename = filename
             self.n_rows = n_rows
             self.data_sha256 = data_sha256
-            self.spectral_data_filenames = spectral_data_filenames
+            self.array_data_filenames = array_data_filenames
 
         super().clean(*args, **kwargs)
 
@@ -182,9 +182,9 @@ def meta_info(self, **kwargs):
                     app_version=self.app_version,
                     id=str(self.id),
                     n_rows=self.n_rows,
-                    n_spectral_data_files=len(self.spectral_data_filenames),
+                    n_array_data_files=len(self.array_data_filenames),
                     timestamp=str(datetime.datetime.now()),
-                    spectral_data_filenames=self.spectral_data_filenames)
+                    array_data_filenames=self.array_data_filenames)
         info.update(kwargs)
         return info
 

biodb/apps/catalog/tests/conftest.py

@@ -5,7 +5,7 @@
 
 from catalog.models import Dataset
 from uploader.tests.conftest import bio_sample_types, centers, instruments, mock_data_from_files, observables, \
-    SimpleQueryFactory, spectra_measurement_types, sql_views, mock_data  # noqa: F401
+    SimpleQueryFactory, array_measurement_types, sql_views, mock_data  # noqa: F401
 from user.models import Center as UserCenter
 
 
@@ -23,7 +23,7 @@ def staffuser(centers):  # noqa: F811
     return User.objects.create(username="staff",
                                email="[email protected]",
                                password="secret",
-                               center=UserCenter.objects.get(name="jhu"),
+                               center=UserCenter.objects.get(name="JHU"),
                                is_staff=True,
                                is_superuser=False)
 
@@ -33,7 +33,7 @@ def cataloguser(centers):  # noqa: F811
     return User.objects.create(username="analyst",
                                email="[email protected]",
                                password="secret",
-                               center=UserCenter.objects.get(name="jhu"),
+                               center=UserCenter.objects.get(name="JHU"),
                                is_staff=True,
                                is_superuser=False,
                                is_catalogviewer=True)
@@ -44,7 +44,7 @@ def superuser(centers):  # noqa: F811
     return User.objects.create(username="admin",
                                email="[email protected]",
                                password="secret",
-                               center=UserCenter.objects.get(name="jhu"),
+                               center=UserCenter.objects.get(name="JHU"),
                                is_staff=True,
                                is_superuser=True)
 

biodb/apps/catalog/tests/test_datasets.py

@@ -8,7 +8,7 @@
 from biodb import __version__
 from catalog.models import Dataset
 from explorer.models import Query
-from uploader.models import SpectralData
+from uploader.models import ArrayData
 
 
 @pytest.mark.django_db(databases=["default", "bsr"])
@@ -49,7 +49,7 @@ def test_files(self, saved_dataset):
                 namelist = z.namelist()
                 namelist.remove(str(Path(saved_dataset.name).with_suffix(file_ext)))
                 namelist.remove("INFO.json")
-                data_dir = Path(SpectralData.UPLOAD_DIR)
+                data_dir = Path(ArrayData.UPLOAD_DIR)
                 for file in namelist:
                     assert Path(file).parent == data_dir
 

biodb/apps/uploader/admin.py

@@ -11,8 +11,8 @@
 from nested_admin import NestedStackedInline, NestedTabularInline, NestedModelAdmin
 
 from biodb.util import to_bool
-from .models import BioSample, Observable, Instrument, Patient, SpectralData, Observation, UploadedFile, Visit,\
-    QCAnnotator, QCAnnotation, Center, get_center, BioSampleType, SpectraMeasurementType
+from .models import BioSample, Observable, Instrument, Patient, ArrayData, Observation, UploadedFile, Visit,\
+    QCAnnotator, QCAnnotation, Center, get_center, BioSampleType, ArrayMeasurementType
 from uploader.forms import ModelForm
 from user.admin import CenterAdmin as UserCenterAdmin
 
@@ -127,10 +127,10 @@ def formfield_for_foreignkey(self, db_field, request, **kwargs):
             field.queryset = field.queryset.filter(visit__center=center)
         elif db_field.name == "bio_sample":
             field.queryset = field.queryset.filter(visit__patient__center=center)
-        elif db_field.name == "spectral_data":
+        elif db_field.name == "array_data":
             field.queryset = field.queryset.filter(bio_sample__visit__patient__center=center)
         elif db_field.name == "qc_annotation":
-            field.queryset = field.queryset.filter(spectral_data__bio_sample__visit__patient__center=center)
+            field.queryset = field.queryset.filter(array_data__bio_sample__visit__patient__center=center)
         elif db_field.name in ("annotator", "measurement_type", "sample_type"):
             # These aren't limited/restricted by center.
             pass
@@ -142,7 +142,7 @@ def formfield_for_foreignkey(self, db_field, request, **kwargs):
 
 
 admin.site.register(BioSampleType)
-admin.site.register(SpectraMeasurementType)
+admin.site.register(ArrayMeasurementType)
 
 
 @admin.register(Instrument)
@@ -163,18 +163,6 @@ class InstrumentAdmin(RestrictedByCenterMixin, ModelAdmin):
                            "center"]
             }
         ),
-        (
-            "Spectrometer",
-            {
-                "fields": ["spectrometer_manufacturer", "spectrometer_model", "spectrometer_serial_number"],
-            }
-        ),
-        (
-            "Laser",
-            {
-                "fields": ["laser_manufacturer", "laser_model", "laser_serial_number"],
-            }
-        ),
         (
             "More Details",
             {
@@ -218,7 +206,7 @@ class UploadedFileAdmin(RestrictedByCenterMixin, ModelAdmin):
     form = UploadedFileForm
     search_fields = ["created_at"]
     search_help_text = "Creation timestamp"
-    list_display = ["pk", "created_at", "meta_data_file", "spectral_data_file", "center"]
+    list_display = ["pk", "created_at", "meta_data_file", "array_data_file", "center"]
     readonly_fields = ["created_at", "updated_at"]  # TODO: Might need specific user group.
     date_hierarchy = "created_at"
     ordering = ("-updated_at",)
@@ -246,13 +234,13 @@ def get_extra(self, request, obj=None, **kwargs):
 @admin.register(QCAnnotation)
 class QCAnnotationAdmin(RestrictedByCenterMixin, ModelAdmin):
     search_fields = ["annotator__name",
-                     "spectral_data__bio_sample__visit__patient__patient_id",
-                     "spectral_data__bio_sample__visit__patient__patient_cid"]
+                     "array_data__bio_sample__visit__patient__patient_id",
+                     "array_data__bio_sample__visit__patient__patient_cid"]
     search_help_text = "Annotator Name, Patient ID or CID"
     readonly_fields = ("value", "created_at", "updated_at")  # TODO: Might need specific user group for timestamps.
     list_display = ["annotator_name", "value", "annotator_value_type", "updated_at"]
     ordering = ("-updated_at",)
-    list_filter = ("spectral_data__bio_sample__visit__patient__center", "annotator__name")
+    list_filter = ("array_data__bio_sample__visit__patient__center", "annotator__name")
 
     @admin.display
     def annotator_name(self, obj):
@@ -268,7 +256,7 @@ def get_queryset(self, request):
         if request.user.is_superuser:
             return qs
         center = Center.objects.get(pk=request.user.center.pk)
-        return qs.filter(spectral_data__bio_sample__visit__patient__center=center)
+        return qs.filter(array_data__bio_sample__visit__patient__center=center)
 
 
 @admin.register(QCAnnotator)
@@ -452,7 +440,7 @@ class ObservationAdmin(ObservationMixin, RestrictedByCenterMixin, NestedModelAdm
     list_display = ["patient_id", "observable_name", "visit"]
 
 
-class SpectralDataMixin:
+class ArrayDataMixin:
     ordering = ("-updated_at",)
     readonly_fields = ["created_at", "updated_at"]
 
@@ -468,8 +456,6 @@ class SpectralDataMixin:
             {
                 "fields": ["measurement_id",
                            "measurement_type",
-                           "atr_crystal",
-                           "n_coadditions",
                            "acquisition_time",
                            "resolution",
                            "power",
@@ -479,16 +465,6 @@ class SpectralDataMixin:
                            "date"],
             }
         ),
-        (
-            "SERS Details",
-            {
-                "classes": ["collapse"],
-                "fields": ["sers_description",
-                           "sers_particle_material",
-                           "sers_particle_size",
-                           "sers_particle_concentration"],
-            }
-        ),
         (
             "More Details",
             {
@@ -510,8 +486,8 @@ def get_queryset(self, request):
         return qs.filter(bio_sample__visit__patient__center=Center.objects.get(pk=request.user.center.pk))
 
 
-@admin.register(SpectralData)
-class SpectralDataAdmin(SpectralDataMixin, RestrictedByCenterMixin, NestedModelAdmin):
+@admin.register(ArrayData)
+class ArrayDataAdmin(ArrayDataMixin, RestrictedByCenterMixin, NestedModelAdmin):
     search_fields = ["bio_sample__visit__patient__patient_id", "bio_sample__visit__patient__patient_cid"]
     search_help_text = "Patient ID or CID"
     readonly_fields = ["created_at", "updated_at"]  # TODO: Might need specific user group.
@@ -525,20 +501,20 @@ class SpectralDataAdmin(SpectralDataMixin, RestrictedByCenterMixin, NestedModelA
                    "bio_sample__visit__observation__observable")
 
 
-class SpectralDataAdminWithInlines(SpectralDataAdmin):
+class ArrayDataAdminWithInlines(ArrayDataAdmin):
     inlines = [QCAnnotationInline]
 
 
-class SpectralDataInline(SpectralDataMixin, RestrictedByCenterMixin, NestedStackedInline):
-    model = SpectralData
+class ArrayDataInline(ArrayDataMixin, RestrictedByCenterMixin, NestedStackedInline):
+    model = ArrayData
     extra = 1
     min_num = 0
     show_change_link = True
     fk_name = "bio_sample"
 
     def get_extra(self, request, obj=None, **kwargs):
         # Only display inlines for those that exist, i.e., no expanded extras (if they exist).
-        return 0 if obj and obj.pk and obj.spectral_data.count() else self.extra
+        return 0 if obj and obj.pk and obj.array_data.count() else self.extra
 
 
 class BioSampleMixin:
@@ -600,7 +576,7 @@ class BioSampleAdmin(BioSampleMixin, RestrictedByCenterMixin, NestedModelAdmin):
 
 
 class BioSampleAdminWithInlines(BioSampleAdmin):
-    inlines = [SpectralDataInline]
+    inlines = [ArrayDataInline]
 
 
 class BioSampleInline(BioSampleMixin, RestrictedByCenterMixin, NestedStackedInline):
@@ -609,7 +585,7 @@ class BioSampleInline(BioSampleMixin, RestrictedByCenterMixin, NestedStackedInli
     min_num = 0
     show_change_link = True
     fk_name = "visit"
-    inlines = [SpectralDataInline]
+    inlines = [ArrayDataInline]
 
     def get_extra(self, request, obj=None, **kwargs):
         # Only display inlines for those that exist, i.e., no expanded extras (if they exist).
@@ -796,7 +772,7 @@ class DataAdminSite(admin.AdminSite):
                    Visit,
                    Observation,
                    BioSample,
-                   SpectralData,
+                   ArrayData,
                    UploadedFile
                    ]
 
@@ -815,7 +791,7 @@ def get_app_list(self, request, app_label=None):
 data_admin.register(Visit, admin_class=VisitAdminWithInlines)
 data_admin.register(Observation, admin_class=ObservationAdmin)
 data_admin.register(BioSample, admin_class=BioSampleAdminWithInlines)
-data_admin.register(SpectralData, admin_class=SpectralDataAdminWithInlines)
+data_admin.register(ArrayData, admin_class=ArrayDataAdminWithInlines)
 data_admin.register(UploadedFile, admin_class=UploadedFileAdmin)
 # data_admin.register(Instrument, admin_class=InstrumentAdmin)
 # data_admin.register(QCAnnotation, admin_class=QCAnnotationAdmin)

biodb/apps/uploader/base_models.py

@@ -20,7 +20,7 @@ def get_column_names(cls, help_text=False):
                    "data",
                    "date",
                    "id",
-                   "spectral data file",
+                   "array data file",
                    "updated at"}
 
         if hasattr(cls, "parse_fields_from_pandas_series"):  # Only models with this func have bulk data upload columns.