Skip to content

Commit 7e0ffbe

Browse files
kevglisskevgliss
authored
Enhancement/signal search filters (Netflix#2976)
* Migrating signal snooze and deduplicate filters to search filters
1 parent 4182951 commit 7e0ffbe

36 files changed

+2207
-655
lines changed

‎src/dispatch/case/models.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -169,7 +169,7 @@ class SignalInstanceRead(DispatchBase):
169169
entities: Optional[List[EntityRead]] = []
170170
tags: Optional[List[TagRead]] = []
171171
raw: Any
172-
fingerprint: str
172+
fingerprint: Optional[str]
173173
created_at: datetime
174174

175175

‎src/dispatch/database/revisions/tenant/versions/2023-02-13_93b517de08e2.py

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
"""Allows many to many signal filters
2+
3+
Revision ID: 93b517de08e2
4+
Revises: b168b50764c7
5+
Create Date: 2023-02-13 15:19:36.921571
6+
7+
"""
8+
from alembic import op
9+
import sqlalchemy as sa
10+
11+
# revision identifiers, used by Alembic.
12+
revision = "93b517de08e2"
13+
down_revision = "b168b50764c7"
14+
branch_labels = None
15+
depends_on = None
16+
17+
18+
def upgrade():
19+
# ### commands auto generated by Alembic - please adjust! ###
20+
op.create_table(
21+
"assoc_signal_filters",
22+
sa.Column("signal_id", sa.Integer(), nullable=False),
23+
sa.Column("signal_filter_id", sa.Integer(), nullable=False),
24+
sa.ForeignKeyConstraint(["signal_filter_id"], ["signal_filter.id"], ondelete="CASCADE"),
25+
sa.ForeignKeyConstraint(["signal_id"], ["signal.id"], ondelete="CASCADE"),
26+
sa.PrimaryKeyConstraint("signal_id", "signal_filter_id"),
27+
)
28+
op.drop_constraint("signal_filter_signal_id_fkey", "signal_filter", type_="foreignkey")
29+
op.drop_column("signal_filter", "signal_id")
30+
# ### end Alembic commands ###
31+
32+
33+
def downgrade():
34+
# ### commands auto generated by Alembic - please adjust! ###
35+
op.add_column(
36+
"signal_filter", sa.Column("signal_id", sa.INTEGER(), autoincrement=False, nullable=True)
37+
)
38+
op.create_foreign_key(
39+
"signal_filter_signal_id_fkey", "signal_filter", "signal", ["signal_id"], ["id"]
40+
)
41+
op.drop_table("assoc_signal_filters")
42+
# ### end Alembic commands ###

‎src/dispatch/database/revisions/tenant/versions/2023-02-13_b168b50764c7.py

+254
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,254 @@
1+
"""Moves signal processing to filter approach.
2+
3+
Revision ID: b168b50764c7
4+
Revises: 8746b4e292d2
5+
Create Date: 2023-02-13 13:56:48.032074
6+
7+
"""
8+
from alembic import op
9+
import sqlalchemy as sa
10+
import sqlalchemy_utils
11+
from sqlalchemy.dialects import postgresql
12+
13+
# revision identifiers, used by Alembic.
14+
revision = "b168b50764c7"
15+
down_revision = "8746b4e292d2"
16+
branch_labels = None
17+
depends_on = None
18+
19+
20+
def upgrade():
21+
# ### commands auto generated by Alembic - please adjust! ###
22+
op.create_table(
23+
"signal_filter",
24+
sa.Column("evergreen", sa.Boolean(), nullable=True),
25+
sa.Column("evergreen_owner", sa.String(), nullable=True),
26+
sa.Column("evergreen_reminder_interval", sa.Integer(), nullable=True),
27+
sa.Column("evergreen_last_reminder_at", sa.DateTime(), nullable=True),
28+
sa.Column("id", sa.Integer(), nullable=False),
29+
sa.Column("name", sa.String(), nullable=True),
30+
sa.Column("description", sa.String(), nullable=True),
31+
sa.Column("expression", sa.JSON(), nullable=False),
32+
sa.Column("mode", sa.String(), nullable=False),
33+
sa.Column("action", sa.String(), nullable=False),
34+
sa.Column("expiration", sa.DateTime(), nullable=True),
35+
sa.Column("window", sa.Integer(), nullable=True),
36+
sa.Column("signal_id", sa.Integer(), nullable=True),
37+
sa.Column("creator_id", sa.Integer(), nullable=True),
38+
sa.Column("search_vector", sqlalchemy_utils.types.ts_vector.TSVectorType(), nullable=True),
39+
sa.Column("project_id", sa.Integer(), nullable=True),
40+
sa.Column("created_at", sa.DateTime(), nullable=True),
41+
sa.Column("updated_at", sa.DateTime(), nullable=True),
42+
sa.ForeignKeyConstraint(
43+
["creator_id"],
44+
["dispatch_core.dispatch_user.id"],
45+
),
46+
sa.ForeignKeyConstraint(["project_id"], ["project.id"], ondelete="CASCADE"),
47+
sa.ForeignKeyConstraint(
48+
["signal_id"],
49+
["signal.id"],
50+
),
51+
sa.PrimaryKeyConstraint("id"),
52+
sa.UniqueConstraint("name", "project_id"),
53+
)
54+
op.create_index(
55+
"signal_filter_search_vector_idx",
56+
"signal_filter",
57+
["search_vector"],
58+
unique=False,
59+
postgresql_using="gin",
60+
)
61+
op.drop_constraint("signal_suppression_rule_id_fkey", "signal", type_="foreignkey")
62+
op.drop_constraint("signal_duplication_rule_id_fkey", "signal", type_="foreignkey")
63+
op.drop_constraint(
64+
"signal_instance_duplication_rule_id_fkey", "signal_instance", type_="foreignkey"
65+
)
66+
op.drop_constraint(
67+
"signal_instance_suppression_rule_id_fkey", "signal_instance", type_="foreignkey"
68+
)
69+
op.drop_table("assoc_duplication_rule_tag_types")
70+
op.drop_table("assoc_suppression_rule_tags")
71+
op.drop_table("assoc_signal_instance_tags")
72+
op.drop_table("duplication_rule")
73+
op.drop_table("suppression_rule")
74+
op.drop_column("signal", "suppression_rule_id")
75+
op.drop_column("signal", "duplication_rule_id")
76+
op.add_column("signal_instance", sa.Column("filter_action", sa.String(), nullable=True))
77+
op.drop_column("signal_instance", "suppression_rule_id")
78+
op.drop_column("signal_instance", "duplication_rule_id")
79+
# ### end Alembic commands ###
80+
81+
82+
def downgrade():
83+
# ### commands auto generated by Alembic - please adjust! ###
84+
op.add_column(
85+
"signal_instance",
86+
sa.Column("duplication_rule_id", sa.INTEGER(), autoincrement=False, nullable=True),
87+
)
88+
op.add_column(
89+
"signal_instance",
90+
sa.Column("suppression_rule_id", sa.INTEGER(), autoincrement=False, nullable=True),
91+
)
92+
op.create_foreign_key(
93+
"signal_instance_suppression_rule_id_fkey",
94+
"signal_instance",
95+
"suppression_rule",
96+
["suppression_rule_id"],
97+
["id"],
98+
)
99+
op.create_foreign_key(
100+
"signal_instance_duplication_rule_id_fkey",
101+
"signal_instance",
102+
"duplication_rule",
103+
["duplication_rule_id"],
104+
["id"],
105+
)
106+
op.drop_column("signal_instance", "filter_action")
107+
op.add_column(
108+
"signal", sa.Column("duplication_rule_id", sa.INTEGER(), autoincrement=False, nullable=True)
109+
)
110+
op.add_column(
111+
"signal", sa.Column("suppression_rule_id", sa.INTEGER(), autoincrement=False, nullable=True)
112+
)
113+
op.create_foreign_key(
114+
"signal_duplication_rule_id_fkey",
115+
"signal",
116+
"duplication_rule",
117+
["duplication_rule_id"],
118+
["id"],
119+
)
120+
op.create_foreign_key(
121+
"signal_suppression_rule_id_fkey",
122+
"signal",
123+
"suppression_rule",
124+
["suppression_rule_id"],
125+
["id"],
126+
)
127+
op.add_column(
128+
"plugin_instance",
129+
sa.Column(
130+
"configuration",
131+
postgresql.JSON(astext_type=sa.Text()),
132+
autoincrement=False,
133+
nullable=True,
134+
),
135+
)
136+
op.drop_index("entity_search_vector_idx", table_name="entity", postgresql_using="gin")
137+
op.create_index("ix_entity_search_vector", "entity", ["search_vector"], unique=False)
138+
op.create_table(
139+
"service_incident",
140+
sa.Column("incident_id", sa.INTEGER(), autoincrement=False, nullable=False),
141+
sa.Column("service_id", sa.INTEGER(), autoincrement=False, nullable=False),
142+
sa.ForeignKeyConstraint(
143+
["incident_id"], ["incident.id"], name="service_incident_incident_id_fkey"
144+
),
145+
sa.ForeignKeyConstraint(
146+
["service_id"], ["service.id"], name="service_incident_service_id_fkey"
147+
),
148+
sa.PrimaryKeyConstraint("incident_id", "service_id", name="service_incident_pkey"),
149+
)
150+
op.create_table(
151+
"assoc_suppression_rule_tags",
152+
sa.Column("suppression_rule_id", sa.INTEGER(), autoincrement=False, nullable=False),
153+
sa.Column("tag_id", sa.INTEGER(), autoincrement=False, nullable=False),
154+
sa.ForeignKeyConstraint(
155+
["suppression_rule_id"],
156+
["suppression_rule.id"],
157+
name="assoc_suppression_rule_tags_suppression_rule_id_fkey",
158+
ondelete="CASCADE",
159+
),
160+
sa.ForeignKeyConstraint(
161+
["tag_id"],
162+
["tag.id"],
163+
name="assoc_suppression_rule_tags_tag_id_fkey",
164+
ondelete="CASCADE",
165+
),
166+
sa.PrimaryKeyConstraint(
167+
"suppression_rule_id", "tag_id", name="assoc_suppression_rule_tags_pkey"
168+
),
169+
)
170+
op.create_table(
171+
"assoc_duplication_rule_tag_types",
172+
sa.Column("duplication_rule_id", sa.INTEGER(), autoincrement=False, nullable=False),
173+
sa.Column("tag_type_id", sa.INTEGER(), autoincrement=False, nullable=False),
174+
sa.ForeignKeyConstraint(
175+
["duplication_rule_id"],
176+
["duplication_rule.id"],
177+
name="assoc_duplication_rule_tag_types_duplication_rule_id_fkey",
178+
ondelete="CASCADE",
179+
),
180+
sa.ForeignKeyConstraint(
181+
["tag_type_id"],
182+
["tag_type.id"],
183+
name="assoc_duplication_rule_tag_types_tag_type_id_fkey",
184+
ondelete="CASCADE",
185+
),
186+
sa.PrimaryKeyConstraint(
187+
"duplication_rule_id", "tag_type_id", name="assoc_duplication_rule_tag_types_pkey"
188+
),
189+
)
190+
op.create_table(
191+
"assoc_signal_instance_tags",
192+
sa.Column("signal_instance_id", postgresql.UUID(), autoincrement=False, nullable=False),
193+
sa.Column("tag_id", sa.INTEGER(), autoincrement=False, nullable=False),
194+
sa.ForeignKeyConstraint(
195+
["signal_instance_id"],
196+
["signal_instance.id"],
197+
name="assoc_signal_instance_tags_signal_instance_id_fkey",
198+
ondelete="CASCADE",
199+
),
200+
sa.ForeignKeyConstraint(
201+
["tag_id"],
202+
["tag.id"],
203+
name="assoc_signal_instance_tags_tag_id_fkey",
204+
ondelete="CASCADE",
205+
),
206+
sa.PrimaryKeyConstraint(
207+
"signal_instance_id", "tag_id", name="assoc_signal_instance_tags_pkey"
208+
),
209+
)
210+
op.create_table(
211+
"suppression_rule",
212+
sa.Column("evergreen", sa.BOOLEAN(), autoincrement=False, nullable=True),
213+
sa.Column("evergreen_owner", sa.VARCHAR(), autoincrement=False, nullable=True),
214+
sa.Column("evergreen_reminder_interval", sa.INTEGER(), autoincrement=False, nullable=True),
215+
sa.Column(
216+
"evergreen_last_reminder_at", postgresql.TIMESTAMP(), autoincrement=False, nullable=True
217+
),
218+
sa.Column("id", sa.INTEGER(), autoincrement=True, nullable=False),
219+
sa.Column("mode", sa.VARCHAR(), autoincrement=False, nullable=False),
220+
sa.Column("expiration", postgresql.TIMESTAMP(), autoincrement=False, nullable=True),
221+
sa.Column("project_id", sa.INTEGER(), autoincrement=False, nullable=True),
222+
sa.ForeignKeyConstraint(
223+
["project_id"],
224+
["project.id"],
225+
name="suppression_rule_project_id_fkey",
226+
ondelete="CASCADE",
227+
),
228+
sa.PrimaryKeyConstraint("id", name="suppression_rule_pkey"),
229+
)
230+
op.create_table(
231+
"duplication_rule",
232+
sa.Column("evergreen", sa.BOOLEAN(), autoincrement=False, nullable=True),
233+
sa.Column("evergreen_owner", sa.VARCHAR(), autoincrement=False, nullable=True),
234+
sa.Column("evergreen_reminder_interval", sa.INTEGER(), autoincrement=False, nullable=True),
235+
sa.Column(
236+
"evergreen_last_reminder_at", postgresql.TIMESTAMP(), autoincrement=False, nullable=True
237+
),
238+
sa.Column("id", sa.INTEGER(), autoincrement=True, nullable=False),
239+
sa.Column("mode", sa.VARCHAR(), autoincrement=False, nullable=False),
240+
sa.Column("project_id", sa.INTEGER(), autoincrement=False, nullable=True),
241+
sa.Column("window", sa.INTEGER(), autoincrement=False, nullable=True),
242+
sa.ForeignKeyConstraint(
243+
["project_id"],
244+
["project.id"],
245+
name="duplication_rule_project_id_fkey",
246+
ondelete="CASCADE",
247+
),
248+
sa.PrimaryKeyConstraint("id", name="duplication_rule_pkey"),
249+
)
250+
op.drop_index(
251+
"signal_filter_search_vector_idx", table_name="signal_filter", postgresql_using="gin"
252+
)
253+
op.drop_table("signal_filter")
254+
# ### end Alembic commands ###

‎src/dispatch/database/service.py

+5-1
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
from dispatch.participant.models import Participant
3434
from dispatch.plugin.models import Plugin, PluginInstance
3535
from dispatch.search.fulltext.composite_search import CompositeSearch
36+
from dispatch.signal.models import SignalInstance
3637
from dispatch.task.models import Task
3738

3839
from .core import Base, get_class_by_tablename, get_db, get_model_name_by_tablename
@@ -347,7 +348,8 @@ def apply_filter_specific_joins(model: Base, filter_spec: dict, query: orm.query
347348
(Incident, "Tag"): (Incident.tags, True),
348349
(Incident, "TagType"): (Incident.tags, True),
349350
(Incident, "Term"): (Incident.terms, True),
350-
(Case, "Tag"): (Case.tags, True),
351+
(SignalInstance, "Entity"): (SignalInstance.entities, True),
352+
(SignalInstance, "EntityType"): (SignalInstance.entities, True),
351353
}
352354
filters = build_filters(filter_spec)
353355
filter_models = get_named_models(filters)[0]
@@ -485,6 +487,8 @@ def search_filter_sort_paginate(
485487
raise ValidationError(
486488
[ErrorWrapper(InvalidFilterError(msg=str(e)), loc="filter")], model=BaseModel
487489
) from None
490+
except Exception as e:
491+
log.exception(e)
488492

489493
if items_per_page == -1:
490494
items_per_page = None

‎src/dispatch/entity/service.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ def update(*, db_session, entity: Entity, entity_in: EntityUpdate) -> Entity:
124124

125125
def delete(*, db_session, entity_id: int):
126126
"""Deletes an existing entity."""
127-
entity = db_session.query(Entity).filter(Entity.id == entity_id).one_or_none()
127+
entity = db_session.query(Entity).filter(Entity.id == entity_id).one()
128128
db_session.delete(entity)
129129
db_session.commit()
130130

‎src/dispatch/entity_type/service.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,6 @@ def update(
9393

9494
def delete(*, db_session: Session, entity_type_id: int) -> None:
9595
"""Deletes an entity type."""
96-
entity_type = db_session.query(EntityType).filter(EntityType.id == entity_type_id)
97-
db_session.delete(entity_type.one_or_none)
96+
entity_type = db_session.query(EntityType).filter(EntityType.id == entity_type_id).one()
97+
db_session.delete(entity_type)
9898
db_session.commit()

‎src/dispatch/enums.py

-7
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,6 @@ def __str__(self) -> str:
66
return str.__str__(self)
77

88

9-
class RuleMode(DispatchEnum):
10-
active = "Active"
11-
monitor = "Monitor"
12-
inactive = "Inactive"
13-
expired = "Expired"
14-
15-
169
class Visibility(DispatchEnum):
1710
open = "Open"
1811
restricted = "Restricted"

‎src/dispatch/main.py

+6
Original file line numberDiff line numberDiff line change
@@ -194,6 +194,12 @@ async def dispatch(
194194
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
195195
content={"detail": [{"msg": "Unknown", "loc": ["Unknown"], "type": "Unknown"}]},
196196
)
197+
except Exception as e:
198+
log.exception(e)
199+
response = JSONResponse(
200+
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
201+
content={"detail": [{"msg": "Unknown", "loc": ["Unknown"], "type": "Unknown"}]},
202+
)
197203

198204
return response
199205

‎src/dispatch/models.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
from typing import Optional
22
from datetime import datetime, timedelta
33

4-
from pydantic import BaseModel
54
from pydantic.fields import Field
65
from pydantic.networks import EmailStr
6+
from pydantic import BaseModel
77
from pydantic.types import conint, constr, SecretStr
88

99
from sqlalchemy import Boolean, Column, DateTime, Integer, String, event, ForeignKey

0 commit comments

Comments
 (0)