Simplifies exclusion handling in data processing

Author: AndrewSazonov

src/easydiffraction/analysis/calculation.py

@@ -31,7 +31,9 @@ def set_calculator(self, engine: str) -> None:
         """
         self._calculator = self.calculator_factory.create_calculator(engine)
 
-    def calculate_structure_factors(self, sample_models: SampleModels, experiments: Experiments) -> Optional[List[Any]]:
+    def calculate_structure_factors(self,
+                                    sample_models: SampleModels,
+                                    experiments: Experiments) -> Optional[List[Any]]:
         """
         Calculate HKL intensities (structure factors) for sample models and experiments.
 
@@ -44,15 +46,17 @@ def calculate_structure_factors(self, sample_models: SampleModels, experiments:
         """
         return self._calculator.calculate_structure_factors(sample_models, experiments)
 
-    def calculate_pattern(self, sample_models: SampleModels, experiment: Experiment) -> np.ndarray:
+    def calculate_pattern(self,
+                          sample_models: SampleModels,
+                          experiment: Experiment) -> np.ndarray:
         """
-        Generate diffraction pattern based on sample models and experiment.
+        Calculate diffraction pattern based on sample models and experiment.
 
         Args:
             sample_models: Collection of sample models.
             experiment: A single experiment object.
 
         Returns:
-            Diffraction pattern generated by the backend calculator.
+            Diffraction pattern calculated by the backend calculator.
         """
         return self._calculator.calculate_pattern(sample_models, experiment)

src/easydiffraction/analysis/calculators/calculator_cryspy.py

@@ -127,6 +127,7 @@ def _recreate_cryspy_dict(self,
 
         cryspy_model_id = f'crystal_{sample_model.name}'
         cryspy_model_dict = cryspy_dict[cryspy_model_id]
+
         # Cell
         cryspy_cell = cryspy_model_dict['unit_cell_parameters']
         cryspy_cell[0] = sample_model.cell.length_a.value
@@ -135,6 +136,7 @@ def _recreate_cryspy_dict(self,
         cryspy_cell[3] = np.deg2rad(sample_model.cell.angle_alpha.value)
         cryspy_cell[4] = np.deg2rad(sample_model.cell.angle_beta.value)
         cryspy_cell[5] = np.deg2rad(sample_model.cell.angle_gamma.value)
+
         # Atomic coordinates
         cryspy_xyz = cryspy_model_dict['atom_fract_xyz']
         for idx, atom_site in enumerate(sample_model.atom_sites):

src/easydiffraction/analysis/minimization.py

@@ -153,11 +153,6 @@ def _residual_function(self,
             y_meas_su: np.ndarray = experiment.datastore.pattern.meas_su
             diff = ((y_meas - y_calc) / y_meas_su)
 
-            # Exclude points that are marked as excluded
-            excluded = experiment.datastore.pattern.excluded
-            if excluded is not None:
-                diff = diff[~excluded]
-
             # Residuals are squared before going into reduced chi-squared
             diff *= np.sqrt(weight)
 

src/easydiffraction/analysis/reliability_factors.py

@@ -132,14 +132,6 @@ def get_reliability_inputs(sample_models: SampleModels,
             if y_meas_su is None:
                 y_meas_su = np.ones_like(y_meas)
 
-            # Exclude points that are marked as excluded
-            excluded = experiment.datastore.pattern.excluded
-            if excluded is not None:
-                y_meas = y_meas[~excluded]
-                y_calc = y_calc[~excluded]
-                if y_meas_su is not None:
-                    y_meas_su = y_meas_su[~excluded]
-
             y_obs_all.extend(y_meas)
             y_calc_all.extend(y_calc)
             y_err_all.extend(y_meas_su)

src/easydiffraction/experiments/collections/excluded_regions.py

@@ -61,10 +61,15 @@ def on_item_added(self, item: ExcludedRegion) -> None:
         """
         experiment = self._parent
         pattern = experiment.datastore.pattern
-        x = pattern.x
 
         # Boolean mask for points within the new excluded region
-        in_region = (x >= item.minimum.value) & (x <= item.maximum.value)
+        in_region = ((pattern.full_x >= item.minimum.value) &
+                     (pattern.full_x <= item.maximum.value))
 
         # Update the exclusion mask
         pattern.excluded[in_region] = True
+
+        # Update the excluded points in the datastore
+        pattern.x = pattern.full_x[~pattern.excluded]
+        pattern.meas = pattern.full_meas[~pattern.excluded]
+        pattern.meas_su = pattern.full_meas_su[~pattern.excluded]

src/easydiffraction/experiments/experiment.py

@@ -286,10 +286,23 @@ def _load_ascii_data_to_experiment(self, data_path: str) -> None:
         sy: np.ndarray = data[:, 2] if data.shape[1] > 2 else np.sqrt(y)
 
         # Attach the data to the experiment's datastore
+
+        # The full pattern data
+        self.datastore.pattern.full_x = x
+        self.datastore.pattern.full_meas = y
+        self.datastore.pattern.full_meas_su = sy
+
+        # The pattern data used for fitting (without excluded points)
+        # This is the same as full_x, full_meas, full_meas_su by default
         self.datastore.pattern.x = x
         self.datastore.pattern.meas = y
         self.datastore.pattern.meas_su = sy
-        self.datastore.pattern.excluded = np.full(x.shape, fill_value=False, dtype=bool)  # No excluded points by default
+
+        # Excluded mask
+        # No excluded points by default
+        self.datastore.pattern.excluded = np.full(x.shape,
+                                                  fill_value=False,
+                                                  dtype=bool)
 
         print(paragraph("Data loaded successfully"))
         print(f"Experiment 🔬 '{self.name}'. Number of data points: {len(x)}")

src/easydiffraction/plotting/plotting.py

@@ -140,15 +140,6 @@ def plot_meas(self,
                                         x_min=x_min,
                                         x_max=x_max)
 
-        # Exclude points based on the pattern's excluded mask
-        if pattern.excluded is not None:
-            excluded = self._filtered_y_array(y_array=pattern.excluded,
-                                              x_array=pattern.x,
-                                              x_min=x_min,
-                                              x_max=x_max)
-            x = x[~excluded]
-            y_meas = y_meas[~excluded]
-
         y_series = [y_meas]
         y_labels = ['meas']
 
@@ -185,15 +176,6 @@ def plot_calc(self,
                                         x_min=x_min,
                                         x_max=x_max)
 
-        # Exclude points based on the pattern's excluded mask
-        if pattern.excluded is not None:
-            excluded = self._filtered_y_array(y_array=pattern.excluded,
-                                              x_array=pattern.x,
-                                              x_min=x_min,
-                                              x_max=x_max)
-            x = x[~excluded]
-            y_calc = y_calc[~excluded]
-
         y_series = [y_calc]
         y_labels = ['calc']
 
@@ -238,16 +220,6 @@ def plot_meas_vs_calc(self,
                                         x_min=x_min,
                                         x_max=x_max)
 
-        # Exclude points based on the pattern's excluded mask
-        if pattern.excluded is not None:
-            excluded = self._filtered_y_array(y_array=pattern.excluded,
-                                              x_array=pattern.x,
-                                              x_min=x_min,
-                                              x_max=x_max)
-            x = x[~excluded]
-            y_meas = y_meas[~excluded]
-            y_calc = y_calc[~excluded]
-
         y_series = [y_meas, y_calc]
         y_labels = ['meas', 'calc']
 

tutorials/cryst-struct_pd-neut-tof_NCAF-WISH.py

@@ -5,7 +5,7 @@
 # structure using time-of-flight neutron powder diffraction data from WISH at
 # ISIS.
 #
-# Two datasets from detector banks 5_6 and 4_7 are used for joint fitting.
+# Two datasets from detector banks 5+6 and 4+7 are used for joint fitting.
 
 # %% [markdown]
 # ## Import Library
@@ -70,10 +70,10 @@
 # ### Create Experiment
 
 # %%
-expt56 = Experiment('wish_5_6', beam_mode='time-of-flight', data_path='data/wish_ncaf_5_6.xye')
+expt56 = Experiment('wish_5_6', beam_mode='time-of-flight', data_path='data/wish_ncaf_5_6.xys')
 
 # %%
-expt47 = Experiment('wish_4_7', beam_mode='time-of-flight', data_path='data/wish_ncaf_4_7.xye')
+expt47 = Experiment('wish_4_7', beam_mode='time-of-flight', data_path='data/wish_ncaf_4_7.xys')
 
 # %% [markdown]
 # ### Set Instrument
@@ -190,6 +190,17 @@
 # %%
 expt47.linked_phases.add('ncaf', scale=2.0)
 
+# %% [markdown]
+# ### Set Excluded Regions
+
+# %%
+expt56.excluded_regions.add(minimum=0, maximum=10010)
+expt56.excluded_regions.add(minimum=100010, maximum=200000)
+
+# %%
+expt47.excluded_regions.add(minimum=0, maximum=10006)
+expt47.excluded_regions.add(minimum=100004, maximum=200000)
+
 # %% [markdown]
 # ## Define Project
 #