Skip to content

Commit e87cd70

Browse files
maikofelix47maikofelix47
committed
Added drug formulation in addition to medication on obs
1 parent 071c27c commit e87cd70

File tree

1 file changed

+339
-0
lines changed

1 file changed

+339
-0
lines changed

‎flat_tables/flat_obs_v1.4.sql

+339
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,339 @@
1+
# This is the ETL table for flat_obs
2+
# obs concept_ids:
3+
4+
# encounter types: 1,2,3,4,5,6,7,8,9,10,13,14,15,17,19,22,23,26,43,47,21
5+
# 1. Replace flat_obs with flat_obs_name
6+
# 2. Replace concept_id in () with concept_id in (obs concept_ids)
7+
# 3. Add column definitions
8+
# 4. Add obs_set column definitions
9+
10+
# v1.1 Notes:
11+
# Added visit_id. This makes it easier to query for visits related indicators
12+
# v1.3 Notes:
13+
# 1. Added updated encounter tracking when updating flat_obs
14+
# 2. Removed voided patients data from flat_obs
15+
# v1.4 Added drug formulation i.e concept_drug
16+
17+
select @table_version := "flat_obs_v1.4";
18+
select @start := now();
19+
20+
set session group_concat_max_len=100000;
21+
select @last_date_created_enc := (select max(date_created) from amrs.encounter);
22+
select @last_date_created_obs := (select max(date_created) from amrs.obs);
23+
select @last_date_created := if(@last_date_created_enc > @last_date_created_obs,@last_date_created_enc,@last_date_created_obs);
24+
select @fake_visit_id := 10000000;
25+
26+
27+
select @boundary := "!!";
28+
select @question_separator := ' ## ';
29+
30+
#delete from flat_log where table_name="flat_obs";
31+
#drop table if exists flat_obs;
32+
create table if not exists flat_obs
33+
(person_id int,
34+
visit_id int,
35+
encounter_id int,
36+
encounter_datetime datetime,
37+
encounter_type int,
38+
location_id int,
39+
obs text,
40+
obs_datetimes text,
41+
max_date_created datetime,
42+
index encounter_id (encounter_id),
43+
index person_date (person_id, encounter_datetime),
44+
index person_enc_id (person_id,encounter_id),
45+
index date_created (max_date_created),
46+
primary key (encounter_id)
47+
);
48+
49+
50+
select @last_date_created_enc := (select max(date_created) from amrs.encounter);
51+
select @last_date_created_obs := (select max(date_created) from amrs.obs);
52+
select @last_date_created := if(@last_date_created_enc > @last_date_created_obs,@last_date_created_enc,@last_date_created_obs);
53+
select @fake_visit_id := 10000000;
54+
55+
56+
# this breaks when replication is down
57+
select @last_update := (select max(date_updated) from flat_log where table_name=@table_version);
58+
59+
# then use the max_date_created from amrs.encounter. This takes about 10 seconds and is better to avoid.
60+
select @last_update :=
61+
if(@last_update is null,
62+
(select max(date_created) from amrs.encounter e join flat_obs using (encounter_id)),
63+
@last_update);
64+
65+
#otherwise set to a date before any encounters had been created (i.g. we will get all encounters)
66+
select @last_update := if(@last_update,@last_update,'1900-01-01');
67+
68+
#select @last_update := "2016-07-06";
69+
70+
drop table if exists voided_obs;
71+
create table voided_obs (index encounter_id (encounter_id), index obs_id (obs_id), index person_datetime (person_id, obs_datetime))
72+
(select person_id, encounter_id, obs_id, obs_datetime, date_voided, concept_id, date_created
73+
from amrs.obs where voided=1 and date_voided > @last_update and date_created <= @last_update);
74+
75+
# find all encounters that were changed after @last_update
76+
drop table if exists encounters_with_updated_data;
77+
create temporary table encounters_with_updated_data
78+
(select
79+
distinct t1.encounter_id
80+
from amrs.encounter t1
81+
join flat_obs t2 using(encounter_id)
82+
where t1.voided=0 and t1.date_changed > @last_update
83+
);
84+
85+
# remove test patients
86+
delete t1
87+
from voided_obs t1
88+
join amrs.person_attribute t2 using (person_id)
89+
where t2.person_attribute_type_id=28 and value='true';
90+
91+
92+
# delete any rows that have voided obs with encounter_id
93+
delete t1
94+
from flat_obs t1
95+
join voided_obs t2 using (encounter_id);
96+
97+
# delete any rows whose encounter changed after last update
98+
delete t1
99+
from flat_obs t1
100+
join encounters_with_updated_data using (encounter_id);
101+
102+
103+
# delete any rows that have a voided obs with no encounter_id
104+
delete t1
105+
from flat_obs t1
106+
join voided_obs t2 on t1.encounter_datetime = t2.obs_datetime and t1.person_id=t2.person_id
107+
where t2.encounter_id is null;
108+
109+
replace into flat_obs
110+
(select
111+
o.person_id,
112+
e.visit_id,
113+
o.encounter_id,
114+
e.encounter_datetime,
115+
e.encounter_type,
116+
e.location_id,
117+
group_concat(
118+
case
119+
when value_drug is not null then concat(@boundary,o.concept_id,'_drug','=',value_drug,@boundary,@question_separator,@boundary,o.concept_id,'=',value_coded,@boundary)
120+
when value_coded is not null then concat(@boundary,o.concept_id,'=',value_coded,@boundary)
121+
when value_numeric is not null then concat(@boundary,o.concept_id,'=',value_numeric,@boundary)
122+
when value_datetime is not null then concat(@boundary,o.concept_id,'=',date(value_datetime),@boundary)
123+
-- when value_boolean is not null then concat(@boundary,o.concept_id,'=',value_boolean,@boundary)
124+
when value_text is not null then concat(@boundary,o.concept_id,'=',value_text,@boundary)
125+
when value_modifier is not null then concat(@boundary,o.concept_id,'=',value_modifier,@boundary)
126+
end
127+
order by o.concept_id,value_coded
128+
separator ' ## '
129+
) as obs,
130+
131+
group_concat(
132+
case
133+
when value_coded is not null or value_numeric is not null or value_datetime is not null or value_text is not null or value_drug is not null or value_modifier is not null
134+
then concat(@boundary,o.concept_id,'=',date(o.obs_datetime),@boundary)
135+
end
136+
order by o.concept_id,value_coded
137+
separator ' ## '
138+
) as obs_datetimes,
139+
max(o.date_created) as max_date_created
140+
141+
from voided_obs v
142+
join amrs.obs o using (encounter_id)
143+
join amrs.encounter e using (encounter_id)
144+
where
145+
o.encounter_id > 1 and o.voided=0
146+
group by encounter_id
147+
);
148+
149+
150+
# Add back obs sets without encounter_ids with voided obs removed
151+
replace into flat_obs
152+
(select
153+
o.person_id,
154+
@fake_visit_id :=@fake_visit_id + 1,
155+
min(o.obs_id) + 100000000 as encounter_id,
156+
o.obs_datetime,
157+
99999 as encounter_type,
158+
null as location_id,
159+
group_concat(
160+
case
161+
when value_drug is not null then concat(@boundary,o.concept_id,'_drug','=',value_drug,@boundary,@question_separator,@boundary,o.concept_id,'=',value_coded,@boundary)
162+
when value_coded is not null then concat(@boundary,o.concept_id,'=',value_coded,@boundary)
163+
when value_numeric is not null then concat(@boundary,o.concept_id,'=',value_numeric,@boundary)
164+
when value_datetime is not null then concat(@boundary,o.concept_id,'=',date(value_datetime),@boundary)
165+
-- when value_boolean is not null then concat(@boundary,o.concept_id,'=',value_boolean,@boundary)
166+
when value_text is not null then concat(@boundary,o.concept_id,'=',value_text,@boundary)
167+
when value_modifier is not null then concat(@boundary,o.concept_id,'=',value_modifier,@boundary)
168+
end
169+
order by o.concept_id,value_coded
170+
separator ' ## '
171+
) as obs,
172+
173+
group_concat(
174+
case
175+
when value_coded is not null or value_numeric is not null or value_datetime is not null or value_text is not null or value_drug is not null or value_modifier is not null
176+
then concat(@boundary,o.concept_id,'=',date(o.obs_datetime),@boundary)
177+
end
178+
order by o.concept_id,value_coded
179+
separator ' ## '
180+
) as obs_datetimes,
181+
max(o.date_created) as max_date_created
182+
183+
from voided_obs v
184+
join amrs.obs o using (person_id, obs_datetime)
185+
where
186+
o.encounter_id is null and voided=0
187+
group by person_id, o.obs_datetime
188+
);
189+
190+
191+
192+
# find all encounters which have new obs after @last_update
193+
drop table if exists encounters_with_new_obs;
194+
create temporary table encounters_with_new_obs
195+
(select
196+
distinct encounter_id
197+
from amrs.obs o
198+
where o.encounter_id > 0
199+
and o.voided=0
200+
and o.date_created > @last_update
201+
);
202+
203+
# Insert newly created obs with encounter_ids
204+
replace into flat_obs
205+
(select
206+
o.person_id,
207+
case
208+
when e.visit_id is not null then e.visit_id else @fake_visit_id :=@fake_visit_id + 1
209+
end as visit_id,
210+
o.encounter_id,
211+
encounter_datetime,
212+
encounter_type,
213+
e.location_id,
214+
group_concat(
215+
case
216+
when value_drug is not null then concat(@boundary,o.concept_id,'_drug','=',value_drug,@boundary,@question_separator,@boundary,o.concept_id,'=',value_coded,@boundary)
217+
when value_coded is not null then concat(@boundary,o.concept_id,'=',value_coded,@boundary)
218+
when value_numeric is not null then concat(@boundary,o.concept_id,'=',value_numeric,@boundary)
219+
when value_datetime is not null then concat(@boundary,o.concept_id,'=',date(value_datetime),@boundary)
220+
-- when value_boolean is not null then concat(@boundary,o.concept_id,'=',value_boolean,@boundary)
221+
when value_text is not null then concat(@boundary,o.concept_id,'=',value_text,@boundary)
222+
when value_modifier is not null then concat(@boundary,o.concept_id,'=',value_modifier,@boundary)
223+
end
224+
order by concept_id,value_coded
225+
separator ' ## '
226+
) as obs,
227+
228+
group_concat(
229+
case
230+
when value_coded is not null or value_numeric is not null or value_datetime is not null or value_text is not null or value_drug is not null or value_modifier is not null
231+
then concat(@boundary,o.concept_id,'=',date(o.obs_datetime),@boundary)
232+
end
233+
order by o.concept_id,value_coded
234+
separator ' ## '
235+
) as obs_datetimes,
236+
max(o.date_created) as max_date_created
237+
238+
from amrs.obs o
239+
join encounters_with_new_obs e1 using (encounter_id)
240+
join amrs.encounter e using (encounter_id)
241+
where o.voided=0
242+
group by o.encounter_id
243+
);
244+
245+
# Insert newly creatred obs without encounter_ids
246+
replace into flat_obs
247+
(select
248+
o.person_id,
249+
@fake_visit_id :=@fake_visit_id + 1 as visit_id,
250+
min(o.obs_id) + 100000000 as encounter_id,
251+
o.obs_datetime,
252+
99999 as encounter_type,
253+
null as location_id,
254+
group_concat(
255+
case
256+
when value_drug is not null then concat(@boundary,o.concept_id,'_drug','=',value_drug,@boundary,@question_separator,@boundary,o.concept_id,'=',value_coded,@boundary)
257+
when value_coded is not null then concat(@boundary,o.concept_id,'=',value_coded,@boundary)
258+
when value_numeric is not null then concat(@boundary,o.concept_id,'=',value_numeric,@boundary)
259+
when value_datetime is not null then concat(@boundary,o.concept_id,'=',date(value_datetime),@boundary)
260+
-- when value_boolean is not null then concat(@boundary,o.concept_id,'=',value_boolean,@boundary)
261+
when value_text is not null then concat(@boundary,o.concept_id,'=',value_text,@boundary)
262+
when value_modifier is not null then concat(@boundary,o.concept_id,'=',value_modifier,@boundary)
263+
end
264+
order by concept_id,value_coded
265+
separator ' ## '
266+
) as obs,
267+
268+
group_concat(
269+
case
270+
when value_coded is not null or value_numeric is not null or value_datetime is not null or value_text is not null or value_drug is not null or value_modifier is not null
271+
then concat(@boundary,o.concept_id,'=',date(o.obs_datetime),@boundary)
272+
end
273+
order by o.concept_id,value_coded
274+
separator ' ## '
275+
) as obs_datetimes,
276+
max(o.date_created) as max_date_created
277+
278+
from amrs.obs o use index (date_created)
279+
where
280+
o.encounter_id is null
281+
and voided=0 and o.date_created > @last_update
282+
group by person_id, o.obs_datetime
283+
);
284+
285+
286+
# Insert obs whose encounters changed after last flat_obs update
287+
replace into flat_obs
288+
(select
289+
o.person_id,
290+
case
291+
when e.visit_id is not null then e.visit_id else @fake_visit_id :=@fake_visit_id + 1
292+
end as visit_id,
293+
o.encounter_id,
294+
encounter_datetime,
295+
encounter_type,
296+
e.location_id,
297+
group_concat(
298+
case
299+
when value_drug is not null then concat(@boundary,o.concept_id,'_drug','=',value_drug,@boundary,@question_separator,@boundary,o.concept_id,'=',value_coded,@boundary)
300+
when value_coded is not null then concat(@boundary,o.concept_id,'=',value_coded,@boundary)
301+
when value_numeric is not null then concat(@boundary,o.concept_id,'=',value_numeric,@boundary)
302+
when value_datetime is not null then concat(@boundary,o.concept_id,'=',date(value_datetime),@boundary)
303+
-- when value_boolean is not null then concat(@boundary,o.concept_id,'=',value_boolean,@boundary)
304+
when value_text is not null then concat(@boundary,o.concept_id,'=',value_text,@boundary)
305+
when value_modifier is not null then concat(@boundary,o.concept_id,'=',value_modifier,@boundary)
306+
end
307+
order by concept_id,value_coded
308+
separator ' ## '
309+
) as obs,
310+
311+
group_concat(
312+
case
313+
when value_coded is not null or value_numeric is not null or value_datetime is not null or value_text is not null or value_drug is not null or value_modifier is not null
314+
then concat(@boundary,o.concept_id,'=',date(o.obs_datetime),@boundary)
315+
end
316+
order by o.concept_id,value_coded
317+
separator ' ## '
318+
) as obs_datetimes,
319+
max(o.date_created) as max_date_created
320+
321+
from amrs.obs o
322+
join encounters_with_updated_data e1 using (encounter_id)
323+
join amrs.encounter e using (encounter_id)
324+
where o.voided=0
325+
group by o.encounter_id
326+
);
327+
328+
329+
# remove voided patients
330+
delete t1
331+
from flat_obs t1
332+
join amrs.person t2 using (person_id)
333+
where t2.voided=1;
334+
335+
drop table voided_obs;
336+
337+
select @end := now();
338+
insert into flat_log values (@start,@last_date_created,@table_version,timestampdiff(second,@start,@end));
339+
select concat(@table_version," : Time to complete: ",timestampdiff(minute, @start, @end)," minutes");

0 commit comments

Comments
 (0)