Skip to content

Commit 1af91a5

Browse files
bsmartradiobsmartradio
committed
Make trailedAssociatorTask
Review update Review Update
1 parent 5d6d4c1 commit 1af91a5

File tree

7 files changed

+278
-17
lines changed

7 files changed

+278
-17
lines changed

‎python/lsst/ap/association/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
# along with this program. If not, see <https://www.gnu.org/licenses/>.
2121

2222
from .version import *
23+
from .trailedSourceFilter import *
2324
from .association import *
2425
from .diaForcedSource import *
2526
from .loadDiaCatalogs import *

‎python/lsst/ap/association/association.py‎

Lines changed: 36 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import lsst.pex.config as pexConfig
3333
import lsst.pipe.base as pipeBase
3434
from lsst.utils.timer import timeMethod
35+
from .trailedSourceFilter import TrailedSourceFilterTask
3536

3637
# Enforce an error for unsafe column/array value setting in pandas.
3738
pd.options.mode.chained_assignment = 'raise'
@@ -40,13 +41,27 @@
4041
class AssociationConfig(pexConfig.Config):
4142
"""Config class for AssociationTask.
4243
"""
44+
4345
maxDistArcSeconds = pexConfig.Field(
4446
dtype=float,
45-
doc='Maximum distance in arcseconds to test for a DIASource to be a '
46-
'match to a DIAObject.',
47+
doc="Maximum distance in arcseconds to test for a DIASource to be a "
48+
"match to a DIAObject.",
4749
default=1.0,
4850
)
4951

52+
trailedSourceFilter = pexConfig.ConfigurableField(
53+
target=TrailedSourceFilterTask,
54+
doc="Subtask to remove long trailed sources based on catalog source "
55+
"morphological measurements.",
56+
)
57+
58+
doTrailedSourceFilter = pexConfig.Field(
59+
doc="Run traildeSourceFilter to remove long trailed sources from "
60+
"output catalog.",
61+
dtype=bool,
62+
default=True,
63+
)
64+
5065

5166
class AssociationTask(pipeBase.Task):
5267
"""Associate DIAOSources into existing DIAObjects.
@@ -60,10 +75,16 @@ class AssociationTask(pipeBase.Task):
6075
ConfigClass = AssociationConfig
6176
_DefaultName = "association"
6277

78+
def __init__(self, *args, **kwargs):
79+
super().__init__(*args, **kwargs)
80+
if self.config.doTrailedSourceFilter:
81+
self.makeSubtask("trailedSourceFilter")
82+
6383
@timeMethod
6484
def run(self,
6585
diaSources,
66-
diaObjects):
86+
diaObjects,
87+
exposure_time=None):
6788
"""Associate the new DiaSources with existing DiaObjects.
6889
6990
Parameters
@@ -72,6 +93,8 @@ def run(self,
7293
New DIASources to be associated with existing DIAObjects.
7394
diaObjects : `pandas.DataFrame`
7495
Existing diaObjects from the Apdb.
96+
exposure_time : `float`, optional
97+
Exposure time from difference image.
7598
7699
Returns
77100
-------
@@ -98,7 +121,16 @@ def run(self,
98121
nUpdatedDiaObjects=0,
99122
nUnassociatedDiaObjects=0)
100123

101-
matchResult = self.associate_sources(diaObjects, diaSources)
124+
if self.config.doTrailedSourceFilter:
125+
diaTrailedResult = self.trailedSourceFilter.run(diaSources, exposure_time)
126+
matchResult = self.associate_sources(diaObjects, diaTrailedResult.diaSources)
127+
128+
self.log.warning("%i DIASources exceed maxTrailLength, dropping "
129+
"from source catalog."
130+
% len(diaTrailedResult.trailedDiaSources))
131+
132+
else:
133+
matchResult = self.associate_sources(diaObjects, diaSources)
102134

103135
mask = matchResult.diaSources["diaObjectId"] != 0
104136

‎python/lsst/ap/association/diaPipe.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,10 @@
2828
Currently loads directly from the Apdb rather than pre-loading.
2929
"""
3030

31+
__all__ = ("DiaPipelineConfig",
32+
"DiaPipelineTask",
33+
"DiaPipelineConnections")
34+
3135
import pandas as pd
3236

3337
import lsst.dax.apdb as daxApdb
@@ -44,10 +48,6 @@
4448
PackageAlertsTask)
4549
from lsst.ap.association.ssoAssociation import SolarSystemAssociationTask
4650

47-
__all__ = ("DiaPipelineConfig",
48-
"DiaPipelineTask",
49-
"DiaPipelineConnections")
50-
5151

5252
class DiaPipelineConnections(
5353
pipeBase.PipelineTaskConnections,
@@ -367,8 +367,8 @@ def run(self,
367367
loaderResult = self.diaCatalogLoader.run(diffIm, self.apdb)
368368

369369
# Associate new DiaSources with existing DiaObjects.
370-
assocResults = self.associator.run(diaSourceTable,
371-
loaderResult.diaObjects)
370+
assocResults = self.associator.run(diaSourceTable, loaderResult.diaObjects,
371+
exposure_time=diffIm.getInfo().getVisitInfo().getExposureTime())
372372
if self.config.doSolarSystemAssociation:
373373
ssoAssocResult = self.solarSystemAssociator.run(
374374
assocResults.unAssocDiaSources,

‎python/lsst/ap/association/trailedSourceFilter.py‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
# This file is part of ap_association.
2+
#
3+
# Developed for the LSST Data Management System.
4+
# This product includes software developed by the LSST Project
5+
# (https://www.lsst.org).
6+
# See the COPYRIGHT file at the top-level directory of this distribution
7+
# for details of code ownership.
8+
#
9+
# This program is free software: you can redistribute it and/or modify
10+
# it under the terms of the GNU General Public License as published by
11+
# the Free Software Foundation, either version 3 of the License, or
12+
# (at your option) any later version.
13+
#
14+
# This program is distributed in the hope that it will be useful,
15+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
16+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17+
# GNU General Public License for more details.
18+
#
19+
# You should have received a copy of the GNU General Public License
20+
# along with this program. If not, see <https://www.gnu.org/licenses/>.
21+
22+
__all__ = ("TrailedSourceFilterTask", "TrailedSourceFilterConfig")
23+
24+
import lsst.pex.config as pexConfig
25+
import lsst.pipe.base as pipeBase
26+
from lsst.utils.timer import timeMethod
27+
28+
29+
class TrailedSourceFilterConfig(pexConfig.Config):
30+
"""Config class for TrailedSourceFilterTask.
31+
"""
32+
33+
maxTrailLength = pexConfig.Field(
34+
dtype=float,
35+
doc="Length of long trailed sources to remove from the input catalog, "
36+
"in arcseconds per second. Default comes from DMTN-199, which "
37+
"requires removal of sources with trails longer than 10 "
38+
"degrees/day, which is 36000/3600/24arcsec/second, or roughly"
39+
"0.416 arcseconds per second.",
40+
default=36000/3600.0/24.0,
41+
)
42+
43+
44+
class TrailedSourceFilterTask(pipeBase.Task):
45+
"""Find trailed sources in DIASources and filter them as per DMTN-199
46+
guidelines.
47+
48+
This task checks the length of trailLength in the DIASource catalog using
49+
a given arcsecond/second rate from maxTrailLength and the exposure time.
50+
The two values are used to calculate the maximum allowed trail length and
51+
filters out any trail longer than the maximum. The maxTrailLength is
52+
outlined in DMTN-199 and determines the default value.
53+
"""
54+
55+
ConfigClass = TrailedSourceFilterConfig
56+
_DefaultName = "trailedSourceFilter"
57+
58+
@timeMethod
59+
def run(self, dia_sources, exposure_time):
60+
"""Remove trailed sources longer than ``config.maxTrailLength`` from
61+
the input catalog.
62+
63+
Parameters
64+
----------
65+
dia_sources : `pandas.DataFrame`
66+
New DIASources to be checked for trailed sources.
67+
exposure_time : `float`
68+
Exposure time from difference image.
69+
70+
Returns
71+
-------
72+
result : `lsst.pipe.base.Struct`
73+
Results struct with components.
74+
75+
- ``dia_sources`` : DIASource table that is free from unwanted
76+
trailed sources. (`pandas.DataFrame`)
77+
78+
- ``trailed_dia_sources`` : DIASources that have trails which
79+
exceed maxTrailLength/second*exposure_time.
80+
(`pandas.DataFrame`)
81+
"""
82+
83+
trail_mask = self._check_dia_source_trail(dia_sources, exposure_time)
84+
85+
return pipeBase.Struct(
86+
diaSources=dia_sources[~trail_mask].reset_index(drop=True),
87+
trailedDiaSources=dia_sources[trail_mask].reset_index(drop=True))
88+
89+
def _check_dia_source_trail(self, dia_sources, exposure_time):
90+
"""Find DiaSources that have long trails.
91+
92+
Creates a mask for sources with lengths greater than 0.416
93+
arcseconds/second multiplied by the exposure time.
94+
95+
Parameters
96+
----------
97+
dia_sources : `pandas.DataFrame`
98+
Input DIASources to check for trail lengths.
99+
exposure_time : `float`
100+
Exposure time from difference image.
101+
102+
Returns
103+
-------
104+
trail_mask : `pandas.DataFrame`
105+
Boolean mask for DIASources which are greater than the
106+
cutoff length.
107+
"""
108+
109+
trail_mask = (dia_sources.loc[:, "trailLength"].values[:]
110+
>= (self.config.maxTrailLength*exposure_time))
111+
112+
return trail_mask

‎tests/test_association_task.py‎

Lines changed: 26 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
import numpy as np
2323
import pandas as pd
2424
import unittest
25-
2625
import lsst.geom as geom
2726
import lsst.utils.tests
2827

@@ -46,20 +45,23 @@ def setUp(self):
4645
self.diaSources = pd.DataFrame(data=[
4746
{"ra": 0.04*idx + scatter*rng.uniform(-1, 1),
4847
"dec": 0.04*idx + scatter*rng.uniform(-1, 1),
49-
"diaSourceId": idx + 1 + self.nObjects, "diaObjectId": 0}
48+
"diaSourceId": idx + 1 + self.nObjects, "diaObjectId": 0, "trailLength": 5.5*idx}
5049
for idx in range(self.nSources)])
5150
self.diaSourceZeroScatter = pd.DataFrame(data=[
5251
{"ra": 0.04*idx,
5352
"dec": 0.04*idx,
54-
"diaSourceId": idx + 1 + self.nObjects, "diaObjectId": 0}
53+
"diaSourceId": idx + 1 + self.nObjects, "diaObjectId": 0, "trailLength": 5.5*idx}
5554
for idx in range(self.nSources)])
55+
self.exposure_time = 30.0
5656

5757
def test_run(self):
5858
"""Test the full task by associating a set of diaSources to
5959
existing diaObjects.
6060
"""
61-
assocTask = AssociationTask()
62-
results = assocTask.run(self.diaSources, self.diaObjects)
61+
config = AssociationTask.ConfigClass()
62+
config.doTrailedSourceFilter = False
63+
assocTask = AssociationTask(config=config)
64+
results = assocTask.run(self.diaSources, self.diaObjects, exposure_time=self.exposure_time)
6365

6466
self.assertEqual(results.nUpdatedDiaObjects, len(self.diaObjects) - 1)
6567
self.assertEqual(results.nUnassociatedDiaObjects, 1)
@@ -75,13 +77,31 @@ def test_run(self):
7577
[0]):
7678
self.assertEqual(test_obj_id, expected_obj_id)
7779

80+
def test_run_trailed_sources(self):
81+
"""Test the full task by associating a set of diaSources to
82+
existing diaObjects when trailed sources are filtered.
83+
84+
This should filter out two of the five sources based on trail length,
85+
leaving one unassociated diaSource and two associated diaSources.
86+
"""
87+
assocTask = AssociationTask()
88+
results = assocTask.run(self.diaSources, self.diaObjects, exposure_time=self.exposure_time)
89+
90+
self.assertEqual(results.nUpdatedDiaObjects, len(self.diaObjects) - 3)
91+
self.assertEqual(results.nUnassociatedDiaObjects, 3)
92+
self.assertEqual(len(results.matchedDiaSources), len(self.diaObjects) - 3)
93+
self.assertEqual(len(results.unAssocDiaSources), 1)
94+
np.testing.assert_array_equal(results.matchedDiaSources["diaObjectId"].values, [1, 2])
95+
np.testing.assert_array_equal(results.unAssocDiaSources["diaObjectId"].values, [0])
96+
7897
def test_run_no_existing_objects(self):
7998
"""Test the run method with a completely empty database.
8099
"""
81100
assocTask = AssociationTask()
82101
results = assocTask.run(
83102
self.diaSources,
84-
pd.DataFrame(columns=["ra", "dec", "diaObjectId"]))
103+
pd.DataFrame(columns=["ra", "dec", "diaObjectId", "trailLength"]),
104+
exposure_time=self.exposure_time)
85105
self.assertEqual(results.nUpdatedDiaObjects, 0)
86106
self.assertEqual(results.nUnassociatedDiaObjects, 0)
87107
self.assertEqual(len(results.matchedDiaSources), 0)

‎tests/test_diaPipe.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,7 @@ def solarSystemAssociator_run(self, unAssocDiaSources, solarSystemObjectTable, d
121121
unAssocDiaSources=MagicMock(spec=pd.DataFrame()))
122122

123123
@lsst.utils.timer.timeMethod
124-
def associator_run(self, table, diaObjects):
124+
def associator_run(self, table, diaObjects, exposure_time=None):
125125
return lsst.pipe.base.Struct(nUpdatedDiaObjects=2, nUnassociatedDiaObjects=3,
126126
matchedDiaSources=MagicMock(spec=pd.DataFrame()),
127127
unAssocDiaSources=MagicMock(spec=pd.DataFrame()))

0 commit comments

Comments
 (0)