From bbc0e2294a5cb92e5af10e9944585d7bd5702aa9 Mon Sep 17 00:00:00 2001 From: Rohaan Sandhu Date: Tue, 8 Jul 2025 11:47:32 -0700 Subject: [PATCH 01/19] Add office-id param to FindMostRecentsInRange, use as another filter --- .../main/java/cwms/cda/api/TimeSeriesRecentController.java | 2 +- .../src/main/java/cwms/cda/data/dao/TimeSeriesDao.java | 2 +- .../src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java | 5 +++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesRecentController.java b/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesRecentController.java index da762699d8..1aa56839d5 100644 --- a/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesRecentController.java +++ b/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesRecentController.java @@ -179,7 +179,7 @@ public void handle(@NotNull Context ctx) { // just group provided latestValues = dao.findRecentsInRange(office, categoryId, groupId, pastLimit, futureLimit, unitSystem); } else { - latestValues = dao.findMostRecentsInRange(tsIds, pastLimit, futureLimit, unitSystem); + latestValues = dao.findMostRecentsInRange(office, tsIds, pastLimit, futureLimit, unitSystem); } String formatHeader = ctx.header(Header.ACCEPT); diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDao.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDao.java index 29be65bb2e..4422f8ca5b 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDao.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDao.java @@ -35,7 +35,7 @@ String getTimeseries(String format, String names, String office, String unit, St List findRecentsInRange(String office, String categoryId, String groupId, Timestamp pastLimit, Timestamp futureLimit, UnitSystem unitSystem); - List findMostRecentsInRange(List tsIds, Timestamp pastLimit, + List findMostRecentsInRange(String office, List tsIds, Timestamp pastLimit, Timestamp futureLimit, UnitSystem unitSystem); } diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index dce0f3344a..9bf23481fa 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -915,7 +915,7 @@ private Tsv buildTsvFromViewRow(usace.cwms.db.jooq.codegen.tables.records.AV_TSV @Override - public List findMostRecentsInRange(List tsIds, Timestamp pastdate, + public List findMostRecentsInRange(String office, List tsIds, Timestamp pastdate, Timestamp futuredate, UnitSystem unitSystem) { List retval = Collections.emptyList(); @@ -957,7 +957,8 @@ public List findMostRecentsInRange(List tsIds, Timestamp pa .and(AV_TSV_DQU.AV_TSV_DQU.DATE_TIME.lt(futuredate)) .and(AV_TSV_DQU.AV_TSV_DQU.DATE_TIME.gt(pastdate)) .and(AV_TSV_DQU.AV_TSV_DQU.START_DATE.le(futuredate)) - .and(AV_TSV_DQU.AV_TSV_DQU.END_DATE.gt(pastdate))); + .and(AV_TSV_DQU.AV_TSV_DQU.END_DATE.gt(pastdate))) + .and(AV_TSV_DQU.AV_TSV_DQU.OFFICE_ID.eq(office)); // We want to use some of the fields from the innerSelect statement in our WHERE clause // Its cleaner if we call them out individually. From eca1f9a11bec880fe90cd4c5c1d608d1aae7a5ec Mon Sep 17 00:00:00 2001 From: Rohaan Sandhu Date: Tue, 8 Jul 2025 13:01:17 -0700 Subject: [PATCH 02/19] Make whereCondition field separate since office may be null --- .../cwms/cda/data/dao/TimeSeriesDaoImpl.java | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index 9bf23481fa..9fa1990795 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -932,6 +932,17 @@ public List findMostRecentsInRange(String office, List tsId DSL.val(unitSystem, String.class)) .as(DEFAULT_UNITS); + Condition whereCondition = AV_CWMS_TS_ID2.CWMS_TS_ID.in(tsIds) + .and(AV_TSV_DQU.AV_TSV_DQU.VALUE.isNotNull()) + .and(AV_TSV_DQU.AV_TSV_DQU.DATE_TIME.lt(futuredate)) + .and(AV_TSV_DQU.AV_TSV_DQU.DATE_TIME.gt(pastdate)) + .and(AV_TSV_DQU.AV_TSV_DQU.START_DATE.le(futuredate)) + .and(AV_TSV_DQU.AV_TSV_DQU.END_DATE.gt(pastdate)); + + if (office != null) { + whereCondition = whereCondition.and(AV_TS_GRP_ASSGN.AV_TS_GRP_ASSGN.DB_OFFICE_ID.eq(office)); + } + SelectConditionStep innerSelect = dsl.select( AV_TSV_DQU.AV_TSV_DQU.OFFICE_ID, AV_TSV_DQU.AV_TSV_DQU.CWMS_TS_ID, @@ -951,14 +962,7 @@ public List findMostRecentsInRange(String office, List tsId .from(AV_TSV_DQU.AV_TSV_DQU.join(AV_CWMS_TS_ID2) .on(AV_TSV_DQU.AV_TSV_DQU.TS_CODE.eq( AV_CWMS_TS_ID2.TS_CODE.cast(Long.class)))) - .where( - AV_CWMS_TS_ID2.CWMS_TS_ID.in(tsIds) - .and(AV_TSV_DQU.AV_TSV_DQU.VALUE.isNotNull()) - .and(AV_TSV_DQU.AV_TSV_DQU.DATE_TIME.lt(futuredate)) - .and(AV_TSV_DQU.AV_TSV_DQU.DATE_TIME.gt(pastdate)) - .and(AV_TSV_DQU.AV_TSV_DQU.START_DATE.le(futuredate)) - .and(AV_TSV_DQU.AV_TSV_DQU.END_DATE.gt(pastdate))) - .and(AV_TSV_DQU.AV_TSV_DQU.OFFICE_ID.eq(office)); + .where(whereCondition); // We want to use some of the fields from the innerSelect statement in our WHERE clause // Its cleaner if we call them out individually. From ddad365758c21a24970c1029c3d1dca93771c483 Mon Sep 17 00:00:00 2001 From: Rohaan Sandhu Date: Wed, 9 Jul 2025 16:02:14 -0700 Subject: [PATCH 03/19] Work in Progress on JOOQ for new query, added in lots of boilerplate, still updating individual pieces --- .../cwms/cda/data/dao/TimeSeriesDaoImpl.java | 275 ++++++++++++------ 1 file changed, 194 insertions(+), 81 deletions(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index 9fa1990795..8b38a52540 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -1,15 +1,10 @@ package cwms.cda.data.dao; import cwms.cda.helpers.DateUtils; -import static org.jooq.impl.DSL.asterisk; -import static org.jooq.impl.DSL.countDistinct; -import static org.jooq.impl.DSL.field; -import static org.jooq.impl.DSL.max; -import static org.jooq.impl.DSL.name; -import static org.jooq.impl.DSL.partitionBy; -import static org.jooq.impl.DSL.select; -import static org.jooq.impl.DSL.selectDistinct; -import static org.jooq.impl.DSL.table; + +import static org.jooq.impl.DSL.*; +import static org.jooq.impl.DSL.inline; +import static org.jooq.impl.DSL.unquotedName; import static usace.cwms.db.jooq.codegen.tables.AV_CWMS_TS_ID2.AV_CWMS_TS_ID2; import static usace.cwms.db.jooq.codegen.tables.AV_TS_EXTENTS_UTC.AV_TS_EXTENTS_UTC; @@ -38,10 +33,7 @@ import java.sql.Connection; import java.sql.SQLException; import java.sql.Timestamp; -import java.time.Duration; -import java.time.Instant; -import java.time.ZoneId; -import java.time.ZonedDateTime; +import java.time.*; import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Arrays; @@ -57,25 +49,7 @@ import java.util.stream.Collectors; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import org.jooq.CommonTableExpression; -import org.jooq.Condition; -import org.jooq.DSLContext; -import org.jooq.Field; -import org.jooq.Record; -import org.jooq.Record1; -import org.jooq.Record3; -import org.jooq.Record4; -import org.jooq.Record7; -import org.jooq.Result; -import org.jooq.SQL; -import org.jooq.SelectConditionStep; -import org.jooq.SelectHavingStep; -import org.jooq.SelectJoinStep; -import org.jooq.SelectSeekStep2; -import org.jooq.Table; -import org.jooq.TableField; -import org.jooq.TableLike; -import org.jooq.TableOnConditionStep; +import org.jooq.*; import org.jooq.conf.ParamType; import org.jooq.exception.DataAccessException; import org.jooq.impl.DSL; @@ -84,12 +58,7 @@ import usace.cwms.db.jooq.codegen.packages.CWMS_LOC_PACKAGE; import usace.cwms.db.jooq.codegen.packages.CWMS_TS_PACKAGE; import usace.cwms.db.jooq.codegen.packages.CWMS_UTIL_PACKAGE; -import usace.cwms.db.jooq.codegen.tables.AV_CWMS_TS_ID; -import usace.cwms.db.jooq.codegen.tables.AV_LOC; -import usace.cwms.db.jooq.codegen.tables.AV_LOC_GRP_ASSGN; -import usace.cwms.db.jooq.codegen.tables.AV_TSV; -import usace.cwms.db.jooq.codegen.tables.AV_TSV_DQU; -import usace.cwms.db.jooq.codegen.tables.AV_TS_GRP_ASSGN; +import usace.cwms.db.jooq.codegen.tables.*; import usace.cwms.db.jooq.codegen.udt.records.ZTSV_ARRAY; import usace.cwms.db.jooq.codegen.udt.records.ZTSV_TYPE; @@ -943,31 +912,72 @@ public List findMostRecentsInRange(String office, List tsId whereCondition = whereCondition.and(AV_TS_GRP_ASSGN.AV_TS_GRP_ASSGN.DB_OFFICE_ID.eq(office)); } - SelectConditionStep innerSelect = dsl.select( - AV_TSV_DQU.AV_TSV_DQU.OFFICE_ID, - AV_TSV_DQU.AV_TSV_DQU.CWMS_TS_ID, - AV_TSV_DQU.AV_TSV_DQU.TS_CODE, - AV_TSV_DQU.AV_TSV_DQU.UNIT_ID, - AV_TSV_DQU.AV_TSV_DQU.DATE_TIME, - AV_TSV_DQU.AV_TSV_DQU.VERSION_DATE, - AV_TSV_DQU.AV_TSV_DQU.DATA_ENTRY_DATE, - AV_TSV_DQU.AV_TSV_DQU.VALUE, - AV_TSV_DQU.AV_TSV_DQU.QUALITY_CODE, - AV_TSV_DQU.AV_TSV_DQU.START_DATE, - AV_TSV_DQU.AV_TSV_DQU.END_DATE, - defUnitsField, - maxDateField, - tsField - ) - .from(AV_TSV_DQU.AV_TSV_DQU.join(AV_CWMS_TS_ID2) - .on(AV_TSV_DQU.AV_TSV_DQU.TS_CODE.eq( - AV_CWMS_TS_ID2.TS_CODE.cast(Long.class)))) - .where(whereCondition); - - // We want to use some of the fields from the innerSelect statement in our WHERE clause - // Its cleaner if we call them out individually. - Field dateTimeField = innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.DATE_TIME); - Field unitField = innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.UNIT_ID); +// // create base_ids +// CommonTableExpression> baseIds = name("base_ids").as( +// selectDistinct(AV_CWMS_TS_ID2.TS_CODE, AV_CWMS_TS_ID2.CWMS_TS_ID) +// .from(AV_CWMS_TS_ID2) +// .where(AV_CWMS_TS_ID2.CWMS_TS_ID.in(tsIds)) +// ); +// +// // create reference to avtsvlimited table +// Table AT_TSV_LIM + + // av tsv limited +// CommonTableExpression atTsv = name("at_tsv_limited").as(select().from(table(name("at_tsv_2023")))); +// +// select( +// field(name("max_values", "cwms_ts_id")), +// field(name("max_values", "date_time")), +// field(name("max_values", "version_date")), +// field(name("max_values", "data_entry_date")), +// field(name("max_values", "quality_code")), +// field(name("max_values", "start_date")), +// field(name("max_values", "end_date")), +// CWMS_UTIL_PACKAGE.call_GET_DEFAULT_UNITS( +// CWMS_TS_PACKAGE.call_GET_BASE_PARAMETER_ID(field(name("max_values", "ts_code"))), +// inline("EN") +// ).as(unquotedName("def_units")), +// field(name("max_values", "date_time")).as(unquotedName("max_date_time")), +// CWMS_UTIL_PACKAGE.call_CONVERT_UNITS( +// field(name("max_values", "value")), +// CWMS_UTIL_PACKAGE.call_GET_DEFAULT_UNITS( +// CWMS_TS_PACKAGE.call_GET_BASE_PARAMETER_ID(field(name("max_values", "ts_code"))), +// inline("SI") +// ), +// CWMS_UTIL_PACKAGE.call_GET_DEFAULT_UNITS( +// CWMS_TS_PACKAGE.call_GET_BASE_PARAMETER_ID(field(name("max_values", "ts_code"))), +// inline("EN") +// ) +// ).as(unquotedName("value_at_max_date")) +// ) +// .from(table(unquotedName("max_values"))) +// .where(field(name("max_values", "date_time")).eq(field(name("max_values", "max_date_time")))) +// +// SelectConditionStep innerSelect = dsl.select( +// AV_TSV_DQU.AV_TSV_DQU.OFFICE_ID, +// AV_TSV_DQU.AV_TSV_DQU.CWMS_TS_ID, +// AV_TSV_DQU.AV_TSV_DQU.TS_CODE, +// AV_TSV_DQU.AV_TSV_DQU.UNIT_ID, +// AV_TSV_DQU.AV_TSV_DQU.DATE_TIME, +// AV_TSV_DQU.AV_TSV_DQU.VERSION_DATE, +// AV_TSV_DQU.AV_TSV_DQU.DATA_ENTRY_DATE, +// AV_TSV_DQU.AV_TSV_DQU.VALUE, +// AV_TSV_DQU.AV_TSV_DQU.QUALITY_CODE, +// AV_TSV_DQU.AV_TSV_DQU.START_DATE, +// AV_TSV_DQU.AV_TSV_DQU.END_DATE, +// defUnitsField, +// maxDateField, +// tsField +// ) +// .from(AV_TSV_DQU.AV_TSV_DQU.join(AV_CWMS_TS_ID2) +// .on(AV_TSV_DQU.AV_TSV_DQU.TS_CODE.eq( +// AV_CWMS_TS_ID2.TS_CODE.cast(Long.class)))) +// .where(whereCondition); +// +// // We want to use some of the fields from the innerSelect statement in our WHERE clause +// // Its cleaner if we call them out individually. +// Field dateTimeField = innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.DATE_TIME); +// Field unitField = innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.UNIT_ID); // We want to return fields from the innerSelect. // Note: Although they are both fields, jOOQ treats @@ -976,24 +986,127 @@ public List findMostRecentsInRange(String office, List tsId // AV_TSV_DQU.AV_TSV_DQU.DATA_ENTRY_DATE // Using the innerSelect field makes DATA_ENTRY_DATE correctly map to Timestamp // and the generated sql refers to columns from the alias_??? table. - Field[] queryFields = new Field[]{ - innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.CWMS_TS_ID), - innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.OFFICE_ID), - innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.TS_CODE), - innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.VERSION_DATE), - innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.DATA_ENTRY_DATE), - innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.VALUE), - innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.QUALITY_CODE), - innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.START_DATE), - innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.END_DATE), - unitField, - dateTimeField, - innerSelect.field(tsField) - }; - - SelectConditionStep query = dsl.select(queryFields) - .from(innerSelect) - .where(dateTimeField.eq(maxDateField).and(unitField.eq(defUnitsField))); +// Field[] queryFields = new Field[]{ +// innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.CWMS_TS_ID), +// innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.OFFICE_ID), +// innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.TS_CODE), +// innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.VERSION_DATE), +// innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.DATA_ENTRY_DATE), +// innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.VALUE), +// innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.QUALITY_CODE), +// innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.START_DATE), +// innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.END_DATE), +// unitField, +// dateTimeField, +// innerSelect.field(tsField) +// }; + +// SelectConditionStep query = dsl.select(queryFields) +// .from(innerSelect) +// .where(dateTimeField.eq(maxDateField).and(unitField.eq(defUnitsField))); +// +// logger.fine(() -> query.getSQL(ParamType.INLINED)); +// retval = query.fetch(r -> buildRecentValue(AV_TSV_DQU.AV_TSV_DQU, r, tsFieldName)); + + // Make a bunch of references idk if all of them are needed + Table AV_CWMS_TS_ID2_TABLE = AV_CWMS_TS_ID2.AV_CWMS_TS_ID2; + Field AV_CWMS_TS_ID2_TS_CODE = AV_CWMS_TS_ID2.AV_CWMS_TS_ID2.TS_CODE; + Field AV_CWMS_TS_ID2_CWMS_TS_ID = AV_CWMS_TS_ID2.AV_CWMS_TS_ID2.CWMS_TS_ID; + Field AV_CWMS_TS_ID2_DB_OFFICE_ID = AV_CWMS_TS_ID2.AV_CWMS_TS_ID2.DB_OFFICE_ID; + + // replace with actual table references? need to be able to have variance on the specific year + // calculated based on timestamps recieved for pastdate and futuredate? + // for now just throwing in 2023 and 2024 + Table AT_TSV_2023_TABLE = table(name("AT_TSV_2023")); + Table AT_TSV_2024_TABLE = table(name("AT_TSV_2024")); + + + Table AT_TS_EXTENTS_TABLE = table(name("AT_TS_EXTENTS")); + Field AT_TS_EXTENTS_TS_CODE = field(name("AT_TS_EXTENTS", "TS_CODE"), String.class); + Field AT_TS_EXTENTS_VERSION_TIME = field(name("AT_TS_EXTENTS", "VERSION_TIME"), Date.class); + Field AT_TS_EXTENTS_EARLIEST_ENTRY_TIME = field(name("AT_TS_EXTENTS", "EARLIEST_ENTRY_TIME"), Date.class); + Field AT_TS_EXTENTS_LATEST_ENTRY_TIME = field(name("AT_TS_EXTENTS", "LATEST_ENTRY_TIME"), Date.class); + + // create baseIds expression + CommonTableExpression baseIds = name("base_ids").as( + select(AV_CWMS_TS_ID2_TS_CODE, AV_CWMS_TS_ID2_CWMS_TS_ID) + .from(AV_CWMS_TS_ID2_TABLE) + .where(AV_CWMS_TS_ID2_CWMS_TS_ID.in(tsIds)) + ); + + // Use the timestamps given to create dates + Date startDate = new Date(pastdate.getTime()); + Date endDate = new Date(pastdate.getTime()); + + // subquery for ts_code in baseIds + Select> tsCodeSubquery = select(baseIds.field(AV_CWMS_TS_ID2_TS_CODE)) + .from(baseIds); + + // create av_tsv_limited expression + CommonTableExpression avTsvLimited = name("av_tsv_limited").as( + select(asterisk()) + .from(AT_TSV_2023_TABLE) + .where(field(AT_TSV_2023_TABLE.getName(), "DATE_TIME", Date.class).between(startDate, endDate)) + .and(field(AT_TSV_2023_TABLE.getName(), "TS_CODE", String.class).in(tsCodeSubquery)) + .unionAll( + select(asterisk()) + .from(AT_TSV_2024_TABLE) + .where(field(AT_TSV_2024_TABLE.getName(), "DATE_TIME", Date.class).between(startDate, endDate)) + .and(field(AT_TSV_2024_TABLE.getName(), "TS_CODE", BigDecimal.class).in(tsCodeSubquery)) + ) + ); + + // create max_values expression + CommonTableExpression maxValues = name("max_values").as( + select( + field(baseIds.getName(), "CWMS_TS_ID", String.class).as("cwms_ts_id"), + field(avTsvLimited.getName(), "TS_CODE", BigDecimal.class).as("ts_code"), + field(avTsvLimited.getName(), "DATE_TIME", Date.class).as("date_time"), + field(avTsvLimited.getName(), "VALUE", BigDecimal.class).as("value"), + field(avTsvLimited.getName(), "VERSION_DATE", Date.class).as("version_date"), + field(avTsvLimited.getName(), "DATA_ENTRY_DATE", Date.class).as("data_entry_date"), + field(avTsvLimited.getName(), "QUALITY_CODE", Integer.class).as("quality_code"), + AT_TS_EXTENTS_EARLIEST_ENTRY_TIME.as("start_date"), + AT_TS_EXTENTS_LATEST_ENTRY_TIME.as("end_date"), + max(field(avTsvLimited.getName(), "DATE_TIME", Date.class)) + .over(partitionBy(field(avTsvLimited.getName(), "TS_CODE", String.class))) + .as("max_date_time") + ) + .from(avTsvLimited) + .join(baseIds).on(field(baseIds.getName(), "TS_CODE", String.class).eq(field(avTsvLimited.getName(), "TS_CODE", String.class))) + .join(AT_TS_EXTENTS_TABLE).on( + AT_TS_EXTENTS_TS_CODE.eq((Select>) field(avTsvLimited.getName(), "TS_CODE", String.class)) + .and(AT_TS_EXTENTS_VERSION_TIME.eq((Select>) field(avTsvLimited.getName(), "VERSION_DATE", Date.class))) + ) + ); + + + Field getDefaultUnits = CWMS_UTIL_PACKAGE.call_GET_DEFAULT_UNITS( + CWMS_TS_PACKAGE.call_GET_BASE_PARAMETER_ID((Number) field(maxValues.getName(), "TS_CODE")), + DSL.val(unitSystem, String.class)) + .as(DEFAULT_UNITS); + + //TODO +// Field convertUnits = CWMS_UTIL_PACKAGE.call_CONVERT_UNITS(field(maxValues.getName(), "VALUE")); + Field convertUnits = null; + + + ResultQuery query = dsl.with(baseIds) + .with(avTsvLimited) + .with(maxValues) + .select( + field(maxValues.getName(), "CWMS_TS_ID", String.class).as("cwms_ts_id"), + field(maxValues.getName(), "DATE_TIME", Date.class).as("date_time"), + field(maxValues.getName(), "VERSION_DATE", Date.class).as("version_date"), + field(maxValues.getName(), "DATA_ENTRY_DATE", Date.class).as("data_entry_date"), + field(maxValues.getName(), "QUALITY_CODE", Integer.class).as("quality_code"), + field(maxValues.getName(), "START_DATE", Date.class).as("start_date"), + field(maxValues.getName(), "END_DATE", Date.class).as("end_date"), + getDefaultUnits.as("def_units"), + field(maxValues.getName(), "DATE_TIME", Date.class).as("max_date_time"), + convertUnits + ) + .from(maxValues); logger.fine(() -> query.getSQL(ParamType.INLINED)); retval = query.fetch(r -> buildRecentValue(AV_TSV_DQU.AV_TSV_DQU, r, tsFieldName)); From e7cf33fa46d49255603991af85fc670d279d567d Mon Sep 17 00:00:00 2001 From: Rohaan Sandhu Date: Thu, 10 Jul 2025 15:35:39 -0700 Subject: [PATCH 04/19] Added most of the new query logic using JOOQ --- .../cwms/cda/data/dao/TimeSeriesDaoImpl.java | 257 +++++++----------- 1 file changed, 91 insertions(+), 166 deletions(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index 8b38a52540..dde8ab5586 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -33,13 +33,14 @@ import java.sql.Connection; import java.sql.SQLException; import java.sql.Timestamp; +import java.sql.Date; import java.time.*; import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.Date; +//import java.util.Date; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -65,6 +66,19 @@ public class TimeSeriesDaoImpl extends JooqDao implements TimeSeriesDao { private static final Logger logger = Logger.getLogger(TimeSeriesDaoImpl.class.getName()); + /** + * String constants for accessing alias table columns in FindMostRecentsRange + */ + private static final String DATE_TIME = "DATE_TIME"; + private static final String VERSION_DATE = "VERSION_DATE"; + private static final String DATA_ENTRY_DATE = "DATA_ENTRY_DATE"; + private static final String QUALITY_CODE = "QUALITY_CODE"; + private static final String START_DATE = "START_DATE"; + private static final String END_DATE = "END_DATE"; + private static final String TS_CODE = "TS_CODE"; + private static final String VALUE = "VALUE"; + private static final String CWMS_TS_ID = "CWMS_TS_ID"; + public static final boolean OVERRIDE_PROTECTION = true; public static final int TS_ID_MISSING_CODE = 20001; public static final String MAX_DATE_TIME = "max_date_time"; @@ -233,9 +247,9 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String ); // Give the TVQ (time, value, quality) columns names - Field dateTimeCol = field("DATE_TIME", Timestamp.class).as("DATE_TIME"); - Field valueCol = field("VALUE", Double.class).as("VALUE"); - Field qualityCol = field("QUALITY_CODE", Integer.class).as("QUALITY_CODE"); + Field dateTimeCol = field(DATE_TIME, Timestamp.class).as(DATE_TIME); + Field valueCol = field(VALUE, Double.class).as(VALUE); + Field qualityCol = field(QUALITY_CODE, Integer.class).as(QUALITY_CODE); Long beginTimeMilli = beginTime.toInstant().toEpochMilli(); Long endTimeMilli = endTime.toInstant().toEpochMilli(); @@ -348,7 +362,7 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String Field qualityNormCol = CWMS_TS_PACKAGE.call_NORMALIZE_QUALITY( DSL.nvl(qualityCol, DSL.inline(5))).as("QUALITY_NORM"); - Field dataEntryDate = field("DATA_ENTRY_DATE", Timestamp.class).as("data_entry_date"); + Field dataEntryDate = field(DATA_ENTRY_DATE, Timestamp.class).as(DATA_ENTRY_DATE); TimeSeries retVal = null; if (pageSize != 0) { @@ -912,201 +926,112 @@ public List findMostRecentsInRange(String office, List tsId whereCondition = whereCondition.and(AV_TS_GRP_ASSGN.AV_TS_GRP_ASSGN.DB_OFFICE_ID.eq(office)); } -// // create base_ids -// CommonTableExpression> baseIds = name("base_ids").as( -// selectDistinct(AV_CWMS_TS_ID2.TS_CODE, AV_CWMS_TS_ID2.CWMS_TS_ID) -// .from(AV_CWMS_TS_ID2) -// .where(AV_CWMS_TS_ID2.CWMS_TS_ID.in(tsIds)) -// ); -// -// // create reference to avtsvlimited table -// Table AT_TSV_LIM - - // av tsv limited -// CommonTableExpression atTsv = name("at_tsv_limited").as(select().from(table(name("at_tsv_2023")))); -// -// select( -// field(name("max_values", "cwms_ts_id")), -// field(name("max_values", "date_time")), -// field(name("max_values", "version_date")), -// field(name("max_values", "data_entry_date")), -// field(name("max_values", "quality_code")), -// field(name("max_values", "start_date")), -// field(name("max_values", "end_date")), -// CWMS_UTIL_PACKAGE.call_GET_DEFAULT_UNITS( -// CWMS_TS_PACKAGE.call_GET_BASE_PARAMETER_ID(field(name("max_values", "ts_code"))), -// inline("EN") -// ).as(unquotedName("def_units")), -// field(name("max_values", "date_time")).as(unquotedName("max_date_time")), -// CWMS_UTIL_PACKAGE.call_CONVERT_UNITS( -// field(name("max_values", "value")), -// CWMS_UTIL_PACKAGE.call_GET_DEFAULT_UNITS( -// CWMS_TS_PACKAGE.call_GET_BASE_PARAMETER_ID(field(name("max_values", "ts_code"))), -// inline("SI") -// ), -// CWMS_UTIL_PACKAGE.call_GET_DEFAULT_UNITS( -// CWMS_TS_PACKAGE.call_GET_BASE_PARAMETER_ID(field(name("max_values", "ts_code"))), -// inline("EN") -// ) -// ).as(unquotedName("value_at_max_date")) -// ) -// .from(table(unquotedName("max_values"))) -// .where(field(name("max_values", "date_time")).eq(field(name("max_values", "max_date_time")))) -// -// SelectConditionStep innerSelect = dsl.select( -// AV_TSV_DQU.AV_TSV_DQU.OFFICE_ID, -// AV_TSV_DQU.AV_TSV_DQU.CWMS_TS_ID, -// AV_TSV_DQU.AV_TSV_DQU.TS_CODE, -// AV_TSV_DQU.AV_TSV_DQU.UNIT_ID, -// AV_TSV_DQU.AV_TSV_DQU.DATE_TIME, -// AV_TSV_DQU.AV_TSV_DQU.VERSION_DATE, -// AV_TSV_DQU.AV_TSV_DQU.DATA_ENTRY_DATE, -// AV_TSV_DQU.AV_TSV_DQU.VALUE, -// AV_TSV_DQU.AV_TSV_DQU.QUALITY_CODE, -// AV_TSV_DQU.AV_TSV_DQU.START_DATE, -// AV_TSV_DQU.AV_TSV_DQU.END_DATE, -// defUnitsField, -// maxDateField, -// tsField -// ) -// .from(AV_TSV_DQU.AV_TSV_DQU.join(AV_CWMS_TS_ID2) -// .on(AV_TSV_DQU.AV_TSV_DQU.TS_CODE.eq( -// AV_CWMS_TS_ID2.TS_CODE.cast(Long.class)))) -// .where(whereCondition); -// -// // We want to use some of the fields from the innerSelect statement in our WHERE clause -// // Its cleaner if we call them out individually. -// Field dateTimeField = innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.DATE_TIME); -// Field unitField = innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.UNIT_ID); - - // We want to return fields from the innerSelect. - // Note: Although they are both fields, jOOQ treats - // innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.DATA_ENTRY_DATE) - // differently than - // AV_TSV_DQU.AV_TSV_DQU.DATA_ENTRY_DATE - // Using the innerSelect field makes DATA_ENTRY_DATE correctly map to Timestamp - // and the generated sql refers to columns from the alias_??? table. -// Field[] queryFields = new Field[]{ -// innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.CWMS_TS_ID), -// innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.OFFICE_ID), -// innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.TS_CODE), -// innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.VERSION_DATE), -// innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.DATA_ENTRY_DATE), -// innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.VALUE), -// innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.QUALITY_CODE), -// innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.START_DATE), -// innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.END_DATE), -// unitField, -// dateTimeField, -// innerSelect.field(tsField) -// }; - -// SelectConditionStep query = dsl.select(queryFields) -// .from(innerSelect) -// .where(dateTimeField.eq(maxDateField).and(unitField.eq(defUnitsField))); -// -// logger.fine(() -> query.getSQL(ParamType.INLINED)); -// retval = query.fetch(r -> buildRecentValue(AV_TSV_DQU.AV_TSV_DQU, r, tsFieldName)); - - // Make a bunch of references idk if all of them are needed - Table AV_CWMS_TS_ID2_TABLE = AV_CWMS_TS_ID2.AV_CWMS_TS_ID2; - Field AV_CWMS_TS_ID2_TS_CODE = AV_CWMS_TS_ID2.AV_CWMS_TS_ID2.TS_CODE; - Field AV_CWMS_TS_ID2_CWMS_TS_ID = AV_CWMS_TS_ID2.AV_CWMS_TS_ID2.CWMS_TS_ID; - Field AV_CWMS_TS_ID2_DB_OFFICE_ID = AV_CWMS_TS_ID2.AV_CWMS_TS_ID2.DB_OFFICE_ID; - - // replace with actual table references? need to be able to have variance on the specific year - // calculated based on timestamps recieved for pastdate and futuredate? - // for now just throwing in 2023 and 2024 - Table AT_TSV_2023_TABLE = table(name("AT_TSV_2023")); - Table AT_TSV_2024_TABLE = table(name("AT_TSV_2024")); - - - Table AT_TS_EXTENTS_TABLE = table(name("AT_TS_EXTENTS")); - Field AT_TS_EXTENTS_TS_CODE = field(name("AT_TS_EXTENTS", "TS_CODE"), String.class); - Field AT_TS_EXTENTS_VERSION_TIME = field(name("AT_TS_EXTENTS", "VERSION_TIME"), Date.class); - Field AT_TS_EXTENTS_EARLIEST_ENTRY_TIME = field(name("AT_TS_EXTENTS", "EARLIEST_ENTRY_TIME"), Date.class); - Field AT_TS_EXTENTS_LATEST_ENTRY_TIME = field(name("AT_TS_EXTENTS", "LATEST_ENTRY_TIME"), Date.class); - - // create baseIds expression + // lines 1 through 8 CommonTableExpression baseIds = name("base_ids").as( - select(AV_CWMS_TS_ID2_TS_CODE, AV_CWMS_TS_ID2_CWMS_TS_ID) - .from(AV_CWMS_TS_ID2_TABLE) - .where(AV_CWMS_TS_ID2_CWMS_TS_ID.in(tsIds)) - ); + selectDistinct(AV_CWMS_TS_ID2.TS_CODE, AV_CWMS_TS_ID2.CWMS_TS_ID) + .from(AV_CWMS_TS_ID2) + .where(AV_CWMS_TS_ID2.CWMS_TS_ID.in(tsIds))); - // Use the timestamps given to create dates + // convert timestamp to date Date startDate = new Date(pastdate.getTime()); - Date endDate = new Date(pastdate.getTime()); + Date endDate = new Date(futuredate.getTime()); - // subquery for ts_code in baseIds - Select> tsCodeSubquery = select(baseIds.field(AV_CWMS_TS_ID2_TS_CODE)) + // helper subquery for SELECT ts_code FROM base_ids + Select> tsCodeSubquery = select(baseIds.field(AV_CWMS_TS_ID2.TS_CODE)) .from(baseIds); - // create av_tsv_limited expression + // references to appropriate year tables + Table AT_TSV_2023_TABLE = table(name("AT_TSV_2023")); + Table AT_TSV_2024_TABLE = table(name("AT_TSV_2024")); + + // create avTsvLimited alias CommonTableExpression avTsvLimited = name("av_tsv_limited").as( select(asterisk()) .from(AT_TSV_2023_TABLE) - .where(field(AT_TSV_2023_TABLE.getName(), "DATE_TIME", Date.class).between(startDate, endDate)) - .and(field(AT_TSV_2023_TABLE.getName(), "TS_CODE", String.class).in(tsCodeSubquery)) + .where(field(AT_TSV_2023_TABLE.getName(), DATE_TIME, Date.class).between(startDate, endDate)) + .and(field(AT_TSV_2023_TABLE.getName(), TS_CODE, BigDecimal.class).in(tsCodeSubquery)) .unionAll( select(asterisk()) .from(AT_TSV_2024_TABLE) - .where(field(AT_TSV_2024_TABLE.getName(), "DATE_TIME", Date.class).between(startDate, endDate)) - .and(field(AT_TSV_2024_TABLE.getName(), "TS_CODE", BigDecimal.class).in(tsCodeSubquery)) + .where(field(AT_TSV_2024_TABLE.getName(), DATE_TIME, Date.class).between(startDate, endDate)) + .and(field(AT_TSV_2024_TABLE.getName(), TS_CODE, BigDecimal.class).in(tsCodeSubquery)) ) ); - // create max_values expression + // Create table and field references for AT_TS_EXTENTS + Table AT_TS_EXTENTS_TABLE = table(name("AT_TS_EXTENTS")); + Field AT_TS_EXTENTS_TS_CODE = field(name("AT_TS_EXTENTS", TS_CODE), BigDecimal.class); + Field AT_TS_EXTENTS_VERSION_TIME = field(name("AT_TS_EXTENTS", "VERSION_TIME"), Date.class); + Field AT_TS_EXTENTS_EARLIEST_ENTRY_TIME = field(name("AT_TS_EXTENTS", "EARLIEST_ENTRY_TIME"), Timestamp.class); + Field AT_TS_EXTENTS_LATEST_ENTRY_TIME = field(name("AT_TS_EXTENTS", "LATEST_ENTRY_TIME"), Timestamp.class); + + // Extract repeated TsCode and DateTime + Field avTsvLimitedTsCode = field(name(avTsvLimited.getName(), TS_CODE), BigDecimal.class); + Field avTsvLimitedDateTime = field(name(avTsvLimited.getName(), DATE_TIME), Date.class); + + // create max_values alias CommonTableExpression maxValues = name("max_values").as( select( - field(baseIds.getName(), "CWMS_TS_ID", String.class).as("cwms_ts_id"), - field(avTsvLimited.getName(), "TS_CODE", BigDecimal.class).as("ts_code"), - field(avTsvLimited.getName(), "DATE_TIME", Date.class).as("date_time"), - field(avTsvLimited.getName(), "VALUE", BigDecimal.class).as("value"), - field(avTsvLimited.getName(), "VERSION_DATE", Date.class).as("version_date"), - field(avTsvLimited.getName(), "DATA_ENTRY_DATE", Date.class).as("data_entry_date"), - field(avTsvLimited.getName(), "QUALITY_CODE", Integer.class).as("quality_code"), - AT_TS_EXTENTS_EARLIEST_ENTRY_TIME.as("start_date"), - AT_TS_EXTENTS_LATEST_ENTRY_TIME.as("end_date"), - max(field(avTsvLimited.getName(), "DATE_TIME", Date.class)) - .over(partitionBy(field(avTsvLimited.getName(), "TS_CODE", String.class))) - .as("max_date_time") + field(name(baseIds.getName(), CWMS_TS_ID), String.class), + avTsvLimitedTsCode, + avTsvLimitedDateTime, + field(name(avTsvLimited.getName(), VALUE), BigDecimal.class), + field(name(avTsvLimited.getName(), VERSION_DATE), Date.class), + field(name(avTsvLimited.getName(), DATA_ENTRY_DATE), Date.class), + field(name(avTsvLimited.getName(), QUALITY_CODE), Integer.class), + AT_TS_EXTENTS_EARLIEST_ENTRY_TIME.as(START_DATE), + AT_TS_EXTENTS_LATEST_ENTRY_TIME.as(END_DATE), + max(avTsvLimitedDateTime) + .over(partitionBy(avTsvLimitedTsCode)) + .as(MAX_DATE_TIME) ) .from(avTsvLimited) - .join(baseIds).on(field(baseIds.getName(), "TS_CODE", String.class).eq(field(avTsvLimited.getName(), "TS_CODE", String.class))) + .join(baseIds).on(field(name(baseIds.getName(), TS_CODE), BigDecimal.class).equal(avTsvLimitedTsCode)) .join(AT_TS_EXTENTS_TABLE).on( - AT_TS_EXTENTS_TS_CODE.eq((Select>) field(avTsvLimited.getName(), "TS_CODE", String.class)) - .and(AT_TS_EXTENTS_VERSION_TIME.eq((Select>) field(avTsvLimited.getName(), "VERSION_DATE", Date.class))) + AT_TS_EXTENTS_TS_CODE.equal(avTsvLimitedTsCode) + .and(AT_TS_EXTENTS_VERSION_TIME + .eq((Date) field(avTsvLimited.getName(), VERSION_DATE, Date.class))) ) ); - Field getDefaultUnits = CWMS_UTIL_PACKAGE.call_GET_DEFAULT_UNITS( - CWMS_TS_PACKAGE.call_GET_BASE_PARAMETER_ID((Number) field(maxValues.getName(), "TS_CODE")), + CWMS_TS_PACKAGE.call_GET_BASE_PARAMETER_ID(field(name(maxValues.getName(), TS_CODE), BigDecimal.class)), DSL.val(unitSystem, String.class)) .as(DEFAULT_UNITS); - //TODO -// Field convertUnits = CWMS_UTIL_PACKAGE.call_CONVERT_UNITS(field(maxValues.getName(), "VALUE")); - Field convertUnits = null; + Field getSIUnits = CWMS_UTIL_PACKAGE.call_GET_DEFAULT_UNITS( + CWMS_TS_PACKAGE.call_GET_BASE_PARAMETER_ID(field(name(maxValues.getName(), TS_CODE), BigDecimal.class)), + DSL.val("SI", String.class)); + Field getENUnits = CWMS_UTIL_PACKAGE.call_GET_DEFAULT_UNITS( + CWMS_TS_PACKAGE.call_GET_BASE_PARAMETER_ID(field(name(maxValues.getName(), TS_CODE), BigDecimal.class)), + DSL.val("EN", String.class)); + Field value = field(name(maxValues.getName(), VALUE), Double.class); + + Field convertUnits = CWMS_UTIL_PACKAGE.call_CONVERT_UNITS( + value, + getSIUnits, + getENUnits + ); + // Final query ResultQuery query = dsl.with(baseIds) .with(avTsvLimited) .with(maxValues) .select( - field(maxValues.getName(), "CWMS_TS_ID", String.class).as("cwms_ts_id"), - field(maxValues.getName(), "DATE_TIME", Date.class).as("date_time"), - field(maxValues.getName(), "VERSION_DATE", Date.class).as("version_date"), - field(maxValues.getName(), "DATA_ENTRY_DATE", Date.class).as("data_entry_date"), - field(maxValues.getName(), "QUALITY_CODE", Integer.class).as("quality_code"), - field(maxValues.getName(), "START_DATE", Date.class).as("start_date"), - field(maxValues.getName(), "END_DATE", Date.class).as("end_date"), - getDefaultUnits.as("def_units"), - field(maxValues.getName(), "DATE_TIME", Date.class).as("max_date_time"), - convertUnits + field(name(maxValues.getName(), CWMS_TS_ID), String.class), + field(name(maxValues.getName(), DATE_TIME), Date.class), + field(name(maxValues.getName(), VERSION_DATE), Date.class), + field(name(maxValues.getName(), DATA_ENTRY_DATE), Date.class), + field(name(maxValues.getName(), QUALITY_CODE), Integer.class), + field(name(maxValues.getName(), START_DATE), Date.class), + field(name(maxValues.getName(), END_DATE), Date.class), + getDefaultUnits.as(DEFAULT_UNITS), + field(name(maxValues.getName(), DATE_TIME), Date.class).as(MAX_DATE_TIME), + convertUnits.as("value_at_max_date") ) - .from(maxValues); + .from(maxValues) + .where(field(name(maxValues.getName(), DATE_TIME)).eq(field(name(maxValues.getName(), MAX_DATE_TIME)))); logger.fine(() -> query.getSQL(ParamType.INLINED)); retval = query.fetch(r -> buildRecentValue(AV_TSV_DQU.AV_TSV_DQU, r, tsFieldName)); From ad996c7ac83b1fb693346ea93733bc74e1853174 Mon Sep 17 00:00:00 2001 From: Rohaan Sandhu Date: Thu, 10 Jul 2025 15:50:20 -0700 Subject: [PATCH 05/19] Add JOOQ query except final query fetch --- .../cwms/cda/data/dao/TimeSeriesDaoImpl.java | 28 +++---------------- 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index dde8ab5586..3a9448837c 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -903,30 +903,9 @@ public List findMostRecentsInRange(String office, List tsId List retval = Collections.emptyList(); if (tsIds != null && !tsIds.isEmpty()) { - String tsFieldName = "TSVIEW_CWMS_TS_ID"; - Field tsField = AV_CWMS_TS_ID2.CWMS_TS_ID.as(tsFieldName); + String tsFieldName = CWMS_TS_ID; - Field maxDateField = max(AV_TSV_DQU.AV_TSV_DQU.DATE_TIME) - .over(partitionBy(AV_TSV_DQU.AV_TSV_DQU.TS_CODE)) - .as(MAX_DATE_TIME); - - Field defUnitsField = CWMS_UTIL_PACKAGE.call_GET_DEFAULT_UNITS( - CWMS_TS_PACKAGE.call_GET_BASE_PARAMETER_ID(AV_TSV_DQU.AV_TSV_DQU.TS_CODE), - DSL.val(unitSystem, String.class)) - .as(DEFAULT_UNITS); - - Condition whereCondition = AV_CWMS_TS_ID2.CWMS_TS_ID.in(tsIds) - .and(AV_TSV_DQU.AV_TSV_DQU.VALUE.isNotNull()) - .and(AV_TSV_DQU.AV_TSV_DQU.DATE_TIME.lt(futuredate)) - .and(AV_TSV_DQU.AV_TSV_DQU.DATE_TIME.gt(pastdate)) - .and(AV_TSV_DQU.AV_TSV_DQU.START_DATE.le(futuredate)) - .and(AV_TSV_DQU.AV_TSV_DQU.END_DATE.gt(pastdate)); - - if (office != null) { - whereCondition = whereCondition.and(AV_TS_GRP_ASSGN.AV_TS_GRP_ASSGN.DB_OFFICE_ID.eq(office)); - } - - // lines 1 through 8 + // create baseIds alias CommonTableExpression baseIds = name("base_ids").as( selectDistinct(AV_CWMS_TS_ID2.TS_CODE, AV_CWMS_TS_ID2.CWMS_TS_ID) .from(AV_CWMS_TS_ID2) @@ -1031,9 +1010,10 @@ public List findMostRecentsInRange(String office, List tsId convertUnits.as("value_at_max_date") ) .from(maxValues) - .where(field(name(maxValues.getName(), DATE_TIME)).eq(field(name(maxValues.getName(), MAX_DATE_TIME)))); + .where(field(name(maxValues.getName(), DATE_TIME), Date.class).eq(field(name(maxValues.getName(), MAX_DATE_TIME), Date.class))); logger.fine(() -> query.getSQL(ParamType.INLINED)); + // TODO: Update query.fetch() and function return since we no longer return an instance of RecentValue retval = query.fetch(r -> buildRecentValue(AV_TSV_DQU.AV_TSV_DQU, r, tsFieldName)); } return retval; From 885368a73912db2a5d301520281c5f0d954e160b Mon Sep 17 00:00:00 2001 From: Rohaan Sandhu Date: Thu, 10 Jul 2025 16:30:29 -0700 Subject: [PATCH 06/19] Fix date imports --- .../cwms/cda/data/dao/TimeSeriesDaoImpl.java | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index 3a9448837c..8ee5a50eb6 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -33,14 +33,13 @@ import java.sql.Connection; import java.sql.SQLException; import java.sql.Timestamp; -import java.sql.Date; import java.time.*; import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; -//import java.util.Date; +import java.util.Date; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -912,8 +911,8 @@ public List findMostRecentsInRange(String office, List tsId .where(AV_CWMS_TS_ID2.CWMS_TS_ID.in(tsIds))); // convert timestamp to date - Date startDate = new Date(pastdate.getTime()); - Date endDate = new Date(futuredate.getTime()); + java.sql.Date startDate = new java.sql.Date(pastdate.getTime()); + java.sql.Date endDate = new java.sql.Date(futuredate.getTime()); // helper subquery for SELECT ts_code FROM base_ids Select> tsCodeSubquery = select(baseIds.field(AV_CWMS_TS_ID2.TS_CODE)) @@ -927,12 +926,12 @@ public List findMostRecentsInRange(String office, List tsId CommonTableExpression avTsvLimited = name("av_tsv_limited").as( select(asterisk()) .from(AT_TSV_2023_TABLE) - .where(field(AT_TSV_2023_TABLE.getName(), DATE_TIME, Date.class).between(startDate, endDate)) + .where(field(AT_TSV_2023_TABLE.getName(), DATE_TIME, java.sql.Date.class).between(startDate, endDate)) .and(field(AT_TSV_2023_TABLE.getName(), TS_CODE, BigDecimal.class).in(tsCodeSubquery)) .unionAll( select(asterisk()) .from(AT_TSV_2024_TABLE) - .where(field(AT_TSV_2024_TABLE.getName(), DATE_TIME, Date.class).between(startDate, endDate)) + .where(field(AT_TSV_2024_TABLE.getName(), DATE_TIME, java.sql.Date.class).between(startDate, endDate)) .and(field(AT_TSV_2024_TABLE.getName(), TS_CODE, BigDecimal.class).in(tsCodeSubquery)) ) ); @@ -940,13 +939,13 @@ public List findMostRecentsInRange(String office, List tsId // Create table and field references for AT_TS_EXTENTS Table AT_TS_EXTENTS_TABLE = table(name("AT_TS_EXTENTS")); Field AT_TS_EXTENTS_TS_CODE = field(name("AT_TS_EXTENTS", TS_CODE), BigDecimal.class); - Field AT_TS_EXTENTS_VERSION_TIME = field(name("AT_TS_EXTENTS", "VERSION_TIME"), Date.class); + Field AT_TS_EXTENTS_VERSION_TIME = field(name("AT_TS_EXTENTS", "VERSION_TIME"), java.sql.Date.class); Field AT_TS_EXTENTS_EARLIEST_ENTRY_TIME = field(name("AT_TS_EXTENTS", "EARLIEST_ENTRY_TIME"), Timestamp.class); Field AT_TS_EXTENTS_LATEST_ENTRY_TIME = field(name("AT_TS_EXTENTS", "LATEST_ENTRY_TIME"), Timestamp.class); // Extract repeated TsCode and DateTime Field avTsvLimitedTsCode = field(name(avTsvLimited.getName(), TS_CODE), BigDecimal.class); - Field avTsvLimitedDateTime = field(name(avTsvLimited.getName(), DATE_TIME), Date.class); + Field avTsvLimitedDateTime = field(name(avTsvLimited.getName(), DATE_TIME), java.sql.Date.class); // create max_values alias CommonTableExpression maxValues = name("max_values").as( @@ -955,8 +954,8 @@ public List findMostRecentsInRange(String office, List tsId avTsvLimitedTsCode, avTsvLimitedDateTime, field(name(avTsvLimited.getName(), VALUE), BigDecimal.class), - field(name(avTsvLimited.getName(), VERSION_DATE), Date.class), - field(name(avTsvLimited.getName(), DATA_ENTRY_DATE), Date.class), + field(name(avTsvLimited.getName(), VERSION_DATE), java.sql.Date.class), + field(name(avTsvLimited.getName(), DATA_ENTRY_DATE), java.sql.Date.class), field(name(avTsvLimited.getName(), QUALITY_CODE), Integer.class), AT_TS_EXTENTS_EARLIEST_ENTRY_TIME.as(START_DATE), AT_TS_EXTENTS_LATEST_ENTRY_TIME.as(END_DATE), @@ -969,7 +968,7 @@ public List findMostRecentsInRange(String office, List tsId .join(AT_TS_EXTENTS_TABLE).on( AT_TS_EXTENTS_TS_CODE.equal(avTsvLimitedTsCode) .and(AT_TS_EXTENTS_VERSION_TIME - .eq((Date) field(avTsvLimited.getName(), VERSION_DATE, Date.class))) + .equal(field(name(avTsvLimited.getName(), VERSION_DATE), java.sql.Date.class))) ) ); @@ -999,18 +998,19 @@ public List findMostRecentsInRange(String office, List tsId .with(maxValues) .select( field(name(maxValues.getName(), CWMS_TS_ID), String.class), - field(name(maxValues.getName(), DATE_TIME), Date.class), - field(name(maxValues.getName(), VERSION_DATE), Date.class), - field(name(maxValues.getName(), DATA_ENTRY_DATE), Date.class), + field(name(maxValues.getName(), DATE_TIME), java.sql.Date.class), + field(name(maxValues.getName(), VERSION_DATE), java.sql.Date.class), + field(name(maxValues.getName(), DATA_ENTRY_DATE), java.sql.Date.class), field(name(maxValues.getName(), QUALITY_CODE), Integer.class), - field(name(maxValues.getName(), START_DATE), Date.class), - field(name(maxValues.getName(), END_DATE), Date.class), + field(name(maxValues.getName(), START_DATE), java.sql.Date.class), + field(name(maxValues.getName(), END_DATE), java.sql.Date.class), getDefaultUnits.as(DEFAULT_UNITS), - field(name(maxValues.getName(), DATE_TIME), Date.class).as(MAX_DATE_TIME), + field(name(maxValues.getName(), DATE_TIME), java.sql.Date.class).as(MAX_DATE_TIME), convertUnits.as("value_at_max_date") ) .from(maxValues) - .where(field(name(maxValues.getName(), DATE_TIME), Date.class).eq(field(name(maxValues.getName(), MAX_DATE_TIME), Date.class))); + .where(field(name(maxValues.getName(), DATE_TIME), java.sql.Date.class) + .eq(field(name(maxValues.getName(), MAX_DATE_TIME), java.sql.Date.class))); logger.fine(() -> query.getSQL(ParamType.INLINED)); // TODO: Update query.fetch() and function return since we no longer return an instance of RecentValue From c650cba8789724d763053bd96f44ae8fa798b2f6 Mon Sep 17 00:00:00 2001 From: Rohaan Sandhu Date: Fri, 11 Jul 2025 09:18:38 -0700 Subject: [PATCH 07/19] Fix field accessing issue --- .../main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index 8ee5a50eb6..f8e04be905 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -926,13 +926,13 @@ public List findMostRecentsInRange(String office, List tsId CommonTableExpression avTsvLimited = name("av_tsv_limited").as( select(asterisk()) .from(AT_TSV_2023_TABLE) - .where(field(AT_TSV_2023_TABLE.getName(), DATE_TIME, java.sql.Date.class).between(startDate, endDate)) - .and(field(AT_TSV_2023_TABLE.getName(), TS_CODE, BigDecimal.class).in(tsCodeSubquery)) + .where(field(name(AT_TSV_2023_TABLE.getName(), DATE_TIME), java.sql.Date.class).between(startDate, endDate)) + .and(field(name(AT_TSV_2023_TABLE.getName(), TS_CODE), BigDecimal.class).in(tsCodeSubquery)) .unionAll( select(asterisk()) .from(AT_TSV_2024_TABLE) - .where(field(AT_TSV_2024_TABLE.getName(), DATE_TIME, java.sql.Date.class).between(startDate, endDate)) - .and(field(AT_TSV_2024_TABLE.getName(), TS_CODE, BigDecimal.class).in(tsCodeSubquery)) + .where(field(name(AT_TSV_2024_TABLE.getName(), DATE_TIME), java.sql.Date.class).between(startDate, endDate)) + .and(field(name(AT_TSV_2024_TABLE.getName(), TS_CODE), BigDecimal.class).in(tsCodeSubquery)) ) ); From 7fa0e91adc397c6df7c6ff2dafcc8f6f24fdd159 Mon Sep 17 00:00:00 2001 From: Rohaan Sandhu Date: Mon, 21 Jul 2025 14:27:17 -0700 Subject: [PATCH 08/19] Add new query, fix year tables, formatting --- .../cwms/cda/data/dao/TimeSeriesDaoImpl.java | 94 ++++++++++++------- 1 file changed, 58 insertions(+), 36 deletions(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index f8e04be905..f3cf48875c 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -77,6 +77,9 @@ public class TimeSeriesDaoImpl extends JooqDao implements TimeSeries private static final String TS_CODE = "TS_CODE"; private static final String VALUE = "VALUE"; private static final String CWMS_TS_ID = "CWMS_TS_ID"; + private static final String AT_TS_EXTENTS = "AT_TS_EXTENTS"; + private static final String VALUE_AT_MAX_DATE = "value_at_max_date"; + private static final String CWMS_20 = "CWMS_20"; public static final boolean OVERRIDE_PROTECTION = true; public static final int TS_ID_MISSING_CODE = 20001; @@ -157,7 +160,7 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String if (includeEntryDate) { Record entryDateSupport = dsl.select(asterisk()).from(table("ALL_TYPES")) .where(field("TYPE_NAME").eq("ZTSV_ENTRY_TYPE")) - .and(field("OWNER").eq("CWMS_20")).fetchOne(); + .and(field("OWNER").eq(CWMS_20)).fetchOne(); if (entryDateSupport == null) { throw new DataAccessException("Data entry date retrieval is not supported by this database"); @@ -914,61 +917,66 @@ public List findMostRecentsInRange(String office, List tsId java.sql.Date startDate = new java.sql.Date(pastdate.getTime()); java.sql.Date endDate = new java.sql.Date(futuredate.getTime()); + // extract year + LocalDate localEndDate = endDate.toLocalDate(); + int year = localEndDate.getYear(); + + // helper subquery for SELECT ts_code FROM base_ids Select> tsCodeSubquery = select(baseIds.field(AV_CWMS_TS_ID2.TS_CODE)) .from(baseIds); - // references to appropriate year tables - Table AT_TSV_2023_TABLE = table(name("AT_TSV_2023")); - Table AT_TSV_2024_TABLE = table(name("AT_TSV_2024")); + // references to appropriate year tables, current year and past year + Table AT_TSV_PREV_YEAR_TABLE = table(name(CWMS_20, "AT_TSV_" + (year - 1))); + Table AT_TSV_CURR_YEAR_TABLE = table(name(CWMS_20, "AT_TSV_" + year)); - // create avTsvLimited alias - CommonTableExpression avTsvLimited = name("av_tsv_limited").as( + // create tsvLimited alias + CommonTableExpression tsvLimited = name("tsv_limited").as( select(asterisk()) - .from(AT_TSV_2023_TABLE) - .where(field(name(AT_TSV_2023_TABLE.getName(), DATE_TIME), java.sql.Date.class).between(startDate, endDate)) - .and(field(name(AT_TSV_2023_TABLE.getName(), TS_CODE), BigDecimal.class).in(tsCodeSubquery)) + .from(AT_TSV_PREV_YEAR_TABLE) + .where(field(name(AT_TSV_PREV_YEAR_TABLE.getName(), DATE_TIME), java.sql.Date.class).between(startDate, endDate)) + .and(field(name(AT_TSV_PREV_YEAR_TABLE.getName(), TS_CODE), BigDecimal.class).in(tsCodeSubquery)) .unionAll( select(asterisk()) - .from(AT_TSV_2024_TABLE) - .where(field(name(AT_TSV_2024_TABLE.getName(), DATE_TIME), java.sql.Date.class).between(startDate, endDate)) - .and(field(name(AT_TSV_2024_TABLE.getName(), TS_CODE), BigDecimal.class).in(tsCodeSubquery)) + .from(AT_TSV_CURR_YEAR_TABLE) + .where(field(name(AT_TSV_CURR_YEAR_TABLE.getName(), DATE_TIME), java.sql.Date.class).between(startDate, endDate)) + .and(field(name(AT_TSV_CURR_YEAR_TABLE.getName(), TS_CODE), BigDecimal.class).in(tsCodeSubquery)) ) ); // Create table and field references for AT_TS_EXTENTS - Table AT_TS_EXTENTS_TABLE = table(name("AT_TS_EXTENTS")); - Field AT_TS_EXTENTS_TS_CODE = field(name("AT_TS_EXTENTS", TS_CODE), BigDecimal.class); - Field AT_TS_EXTENTS_VERSION_TIME = field(name("AT_TS_EXTENTS", "VERSION_TIME"), java.sql.Date.class); - Field AT_TS_EXTENTS_EARLIEST_ENTRY_TIME = field(name("AT_TS_EXTENTS", "EARLIEST_ENTRY_TIME"), Timestamp.class); - Field AT_TS_EXTENTS_LATEST_ENTRY_TIME = field(name("AT_TS_EXTENTS", "LATEST_ENTRY_TIME"), Timestamp.class); + Table AT_TS_EXTENTS_TABLE = table(name(CWMS_20, AT_TS_EXTENTS)); + Field AT_TS_EXTENTS_TS_CODE = field(name(CWMS_20, AT_TS_EXTENTS, TS_CODE), BigDecimal.class); + Field AT_TS_EXTENTS_VERSION_TIME = field(name(CWMS_20, AT_TS_EXTENTS, "VERSION_TIME"), java.sql.Date.class); + Field AT_TS_EXTENTS_EARLIEST_ENTRY_TIME = field(name(CWMS_20, AT_TS_EXTENTS, "EARLIEST_ENTRY_TIME"), Timestamp.class); + Field AT_TS_EXTENTS_LATEST_ENTRY_TIME = field(name(CWMS_20, AT_TS_EXTENTS, "LATEST_ENTRY_TIME"), Timestamp.class); // Extract repeated TsCode and DateTime - Field avTsvLimitedTsCode = field(name(avTsvLimited.getName(), TS_CODE), BigDecimal.class); - Field avTsvLimitedDateTime = field(name(avTsvLimited.getName(), DATE_TIME), java.sql.Date.class); + Field tsvLimitedTsCode = field(name(tsvLimited.getName(), TS_CODE), BigDecimal.class); + Field tsvLimitedDateTime = field(name(tsvLimited.getName(), DATE_TIME), java.sql.Date.class); // create max_values alias CommonTableExpression maxValues = name("max_values").as( select( field(name(baseIds.getName(), CWMS_TS_ID), String.class), - avTsvLimitedTsCode, - avTsvLimitedDateTime, - field(name(avTsvLimited.getName(), VALUE), BigDecimal.class), - field(name(avTsvLimited.getName(), VERSION_DATE), java.sql.Date.class), - field(name(avTsvLimited.getName(), DATA_ENTRY_DATE), java.sql.Date.class), - field(name(avTsvLimited.getName(), QUALITY_CODE), Integer.class), + tsvLimitedTsCode, + tsvLimitedDateTime, + field(name(tsvLimited.getName(), VALUE), BigDecimal.class), + field(name(tsvLimited.getName(), VERSION_DATE), java.sql.Date.class), + field(name(tsvLimited.getName(), DATA_ENTRY_DATE), java.sql.Date.class), + field(name(tsvLimited.getName(), QUALITY_CODE), Integer.class), AT_TS_EXTENTS_EARLIEST_ENTRY_TIME.as(START_DATE), AT_TS_EXTENTS_LATEST_ENTRY_TIME.as(END_DATE), - max(avTsvLimitedDateTime) - .over(partitionBy(avTsvLimitedTsCode)) + max(tsvLimitedDateTime) + .over(partitionBy(tsvLimitedTsCode)) .as(MAX_DATE_TIME) ) - .from(avTsvLimited) - .join(baseIds).on(field(name(baseIds.getName(), TS_CODE), BigDecimal.class).equal(avTsvLimitedTsCode)) + .from(tsvLimited) + .join(baseIds).on(field(name(baseIds.getName(), TS_CODE), BigDecimal.class).equal(tsvLimitedTsCode)) .join(AT_TS_EXTENTS_TABLE).on( - AT_TS_EXTENTS_TS_CODE.equal(avTsvLimitedTsCode) + AT_TS_EXTENTS_TS_CODE.equal(tsvLimitedTsCode) .and(AT_TS_EXTENTS_VERSION_TIME - .equal(field(name(avTsvLimited.getName(), VERSION_DATE), java.sql.Date.class))) + .equal(field(name(tsvLimited.getName(), VERSION_DATE), java.sql.Date.class))) ) ); @@ -993,8 +1001,8 @@ public List findMostRecentsInRange(String office, List tsId ); // Final query - ResultQuery query = dsl.with(baseIds) - .with(avTsvLimited) + SelectConditionStep> query = dsl.with(baseIds) + .with(tsvLimited) .with(maxValues) .select( field(name(maxValues.getName(), CWMS_TS_ID), String.class), @@ -1006,16 +1014,30 @@ public List findMostRecentsInRange(String office, List tsId field(name(maxValues.getName(), END_DATE), java.sql.Date.class), getDefaultUnits.as(DEFAULT_UNITS), field(name(maxValues.getName(), DATE_TIME), java.sql.Date.class).as(MAX_DATE_TIME), - convertUnits.as("value_at_max_date") + convertUnits.as(VALUE_AT_MAX_DATE) ) .from(maxValues) .where(field(name(maxValues.getName(), DATE_TIME), java.sql.Date.class) .eq(field(name(maxValues.getName(), MAX_DATE_TIME), java.sql.Date.class))); logger.fine(() -> query.getSQL(ParamType.INLINED)); - // TODO: Update query.fetch() and function return since we no longer return an instance of RecentValue - retval = query.fetch(r -> buildRecentValue(AV_TSV_DQU.AV_TSV_DQU, r, tsFieldName)); + retval = query.fetch(r -> { + TsvDqu tsv = new TsvDqu.Builder() + .withOfficeId(office) + .withCwmsTsId(r.getValue(CWMS_TS_ID, String.class)) + .withUnitId(r.getValue(DEFAULT_UNITS, String.class)) + .withDateTime(r.getValue(DATE_TIME, java.sql.Date.class)) + .withVersionDate(r.getValue(VERSION_DATE, java.sql.Date.class)) + .withDataEntryDate(r.getValue(DATA_ENTRY_DATE, java.sql.Date.class)) + .withValue(r.getValue(VALUE_AT_MAX_DATE, Double.class)) + .withQualityCode(r.getValue(QUALITY_CODE, Long.class)) + .withStartDate(r.getValue(START_DATE, java.sql.Date.class)) + .withEndDate(r.getValue(END_DATE, java.sql.Date.class)) + .build(); + return new RecentValue(r.getValue(CWMS_TS_ID, String.class), tsv); + }); } + return retval; } From c1050eef3f8769748301dc25f1433047c8aca52e Mon Sep 17 00:00:00 2001 From: Rohaan Sandhu Date: Mon, 21 Jul 2025 15:00:40 -0700 Subject: [PATCH 09/19] Fix imports --- .../cwms/cda/data/dao/TimeSeriesDaoImpl.java | 47 ++++++++++++++++--- 1 file changed, 41 insertions(+), 6 deletions(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index f3cf48875c..600f3541c7 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -2,9 +2,15 @@ import cwms.cda.helpers.DateUtils; -import static org.jooq.impl.DSL.*; -import static org.jooq.impl.DSL.inline; -import static org.jooq.impl.DSL.unquotedName; +import static org.jooq.impl.DSL.asterisk; +import static org.jooq.impl.DSL.countDistinct; +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.max; +import static org.jooq.impl.DSL.name; +import static org.jooq.impl.DSL.partitionBy; +import static org.jooq.impl.DSL.select; +import static org.jooq.impl.DSL.selectDistinct; +import static org.jooq.impl.DSL.table; import static usace.cwms.db.jooq.codegen.tables.AV_CWMS_TS_ID2.AV_CWMS_TS_ID2; import static usace.cwms.db.jooq.codegen.tables.AV_TS_EXTENTS_UTC.AV_TS_EXTENTS_UTC; @@ -33,7 +39,11 @@ import java.sql.Connection; import java.sql.SQLException; import java.sql.Timestamp; -import java.time.*; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Arrays; @@ -49,7 +59,27 @@ import java.util.stream.Collectors; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import org.jooq.*; +import org.jooq.CommonTableExpression; +import org.jooq.Condition; +import org.jooq.DSLContext; +import org.jooq.Field; +import org.jooq.Record; +import org.jooq.Record1; +import org.jooq.Record3; +import org.jooq.Record4; +import org.jooq.Record7; +import org.jooq.Record10; +import org.jooq.Result; +import org.jooq.SQL; +import org.jooq.SelectConditionStep; +import org.jooq.SelectHavingStep; +import org.jooq.SelectJoinStep; +import org.jooq.SelectSeekStep2; +import org.jooq.Table; +import org.jooq.TableField; +import org.jooq.TableLike; +import org.jooq.TableOnConditionStep; +import org.jooq.Select; import org.jooq.conf.ParamType; import org.jooq.exception.DataAccessException; import org.jooq.impl.DSL; @@ -58,7 +88,12 @@ import usace.cwms.db.jooq.codegen.packages.CWMS_LOC_PACKAGE; import usace.cwms.db.jooq.codegen.packages.CWMS_TS_PACKAGE; import usace.cwms.db.jooq.codegen.packages.CWMS_UTIL_PACKAGE; -import usace.cwms.db.jooq.codegen.tables.*; +import usace.cwms.db.jooq.codegen.tables.AV_CWMS_TS_ID; +import usace.cwms.db.jooq.codegen.tables.AV_LOC; +import usace.cwms.db.jooq.codegen.tables.AV_LOC_GRP_ASSGN; +import usace.cwms.db.jooq.codegen.tables.AV_TSV; +import usace.cwms.db.jooq.codegen.tables.AV_TSV_DQU; +import usace.cwms.db.jooq.codegen.tables.AV_TS_GRP_ASSGN; import usace.cwms.db.jooq.codegen.udt.records.ZTSV_ARRAY; import usace.cwms.db.jooq.codegen.udt.records.ZTSV_TYPE; From 7a231d8168ad49e5ccee46c80a96ca171dde09a1 Mon Sep 17 00:00:00 2001 From: Rohaan Sandhu Date: Tue, 22 Jul 2025 10:41:38 -0700 Subject: [PATCH 10/19] Fix unit conversion in new query --- .../cwms/cda/data/dao/TimeSeriesDaoImpl.java | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index 600f3541c7..1e77f7538c 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -115,6 +115,7 @@ public class TimeSeriesDaoImpl extends JooqDao implements TimeSeries private static final String AT_TS_EXTENTS = "AT_TS_EXTENTS"; private static final String VALUE_AT_MAX_DATE = "value_at_max_date"; private static final String CWMS_20 = "CWMS_20"; + private static final String UNIT_ID = "UNIT_ID"; public static final boolean OVERRIDE_PROTECTION = true; public static final int TS_ID_MISSING_CODE = 20001; @@ -944,7 +945,7 @@ public List findMostRecentsInRange(String office, List tsId // create baseIds alias CommonTableExpression baseIds = name("base_ids").as( - selectDistinct(AV_CWMS_TS_ID2.TS_CODE, AV_CWMS_TS_ID2.CWMS_TS_ID) + selectDistinct(AV_CWMS_TS_ID2.TS_CODE, AV_CWMS_TS_ID2.CWMS_TS_ID, AV_CWMS_TS_ID2.UNIT_ID) .from(AV_CWMS_TS_ID2) .where(AV_CWMS_TS_ID2.CWMS_TS_ID.in(tsIds))); @@ -994,6 +995,7 @@ public List findMostRecentsInRange(String office, List tsId CommonTableExpression maxValues = name("max_values").as( select( field(name(baseIds.getName(), CWMS_TS_ID), String.class), + field(name(baseIds.getName(), UNIT_ID), String.class), tsvLimitedTsCode, tsvLimitedDateTime, field(name(tsvLimited.getName(), VALUE), BigDecimal.class), @@ -1015,24 +1017,17 @@ public List findMostRecentsInRange(String office, List tsId ) ); + // Set default units and convert Field getDefaultUnits = CWMS_UTIL_PACKAGE.call_GET_DEFAULT_UNITS( CWMS_TS_PACKAGE.call_GET_BASE_PARAMETER_ID(field(name(maxValues.getName(), TS_CODE), BigDecimal.class)), - DSL.val(unitSystem, String.class)) - .as(DEFAULT_UNITS); + DSL.val(unitSystem, String.class)); - Field getSIUnits = CWMS_UTIL_PACKAGE.call_GET_DEFAULT_UNITS( - CWMS_TS_PACKAGE.call_GET_BASE_PARAMETER_ID(field(name(maxValues.getName(), TS_CODE), BigDecimal.class)), - DSL.val("SI", String.class)); - - Field getENUnits = CWMS_UTIL_PACKAGE.call_GET_DEFAULT_UNITS( - CWMS_TS_PACKAGE.call_GET_BASE_PARAMETER_ID(field(name(maxValues.getName(), TS_CODE), BigDecimal.class)), - DSL.val("EN", String.class)); Field value = field(name(maxValues.getName(), VALUE), Double.class); Field convertUnits = CWMS_UTIL_PACKAGE.call_CONVERT_UNITS( value, - getSIUnits, - getENUnits + field(name(maxValues.getName(), UNIT_ID), String.class), + getDefaultUnits ); // Final query From 7c12054c957bfc4501ac44aa18d782f4dc8dea56 Mon Sep 17 00:00:00 2001 From: Rohaan Sandhu Date: Tue, 22 Jul 2025 10:50:06 -0700 Subject: [PATCH 11/19] Clean up unnecessary helper methods with only one usage --- .../cwms/cda/data/dao/TimeSeriesDaoImpl.java | 45 ++++++++----------- 1 file changed, 18 insertions(+), 27 deletions(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index 1e77f7538c..fed554bdda 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -1051,6 +1051,8 @@ public List findMostRecentsInRange(String office, List tsId .eq(field(name(maxValues.getName(), MAX_DATE_TIME), java.sql.Date.class))); logger.fine(() -> query.getSQL(ParamType.INLINED)); + + // fetch and build records retval = query.fetch(r -> { TsvDqu tsv = new TsvDqu.Builder() .withOfficeId(office) @@ -1071,31 +1073,6 @@ public List findMostRecentsInRange(String office, List tsId return retval; } - - @NotNull - private RecentValue buildRecentValue(AV_TSV_DQU tsvView, Record jrecord, String tsColumnName) { - - TsvDqu tsv = buildTsvDqu(tsvView, jrecord, tsColumnName); - String tsId = jrecord.getValue(tsColumnName, String.class); - return new RecentValue(tsId, tsv); - } - - @NotNull - private TsvDqu buildTsvDqu(AV_TSV_DQU tsvView, Record jrecord, String tsColumnName) { - return new TsvDqu.Builder() - .withOfficeId(jrecord.getValue(tsvView.OFFICE_ID)) - .withCwmsTsId(jrecord.getValue(tsvView.CWMS_TS_ID.as(tsColumnName))) - .withUnitId(jrecord.getValue(tsvView.UNIT_ID)) - .withDateTime(jrecord.getValue(tsvView.DATE_TIME)) - .withVersionDate(jrecord.getValue(tsvView.VERSION_DATE)) - .withDataEntryDate(jrecord.getValue(tsvView.DATA_ENTRY_DATE)) - .withValue(jrecord.getValue(tsvView.VALUE)) - .withQualityCode(jrecord.getValue(tsvView.QUALITY_CODE)) - .withStartDate(jrecord.getValue(tsvView.START_DATE)) - .withEndDate(jrecord.getValue(tsvView.END_DATE)) - .build(); - } - @Override public List findRecentsInRange(String office, String categoryId, String groupId, @NotNull Timestamp pastLimit, @NotNull Timestamp futureLimit, @@ -1162,8 +1139,22 @@ public List findRecentsInRange(String office, String categoryId, St .from(innerSelect) .where(dateTime.eq(maxDateTimeField).and(defUnitsField.eq(unit))) .orderBy(field(AV_TS_GRP_ASSGN.AV_TS_GRP_ASSGN.ATTRIBUTE.getName())) - .fetch(r -> buildRecentValue(tsvView, r, AV_TS_GRP_ASSGN.AV_TS_GRP_ASSGN.TS_ID.getName())) - ; + .fetch(r -> { + String tsId = r.getValue(AV_TS_GRP_ASSGN.AV_TS_GRP_ASSGN.TS_ID.getName(), String.class); + TsvDqu tsv = new TsvDqu.Builder() + .withOfficeId(r.getValue(tsvView.OFFICE_ID)) + .withCwmsTsId(r.getValue(tsvView.CWMS_TS_ID.as(tsId))) + .withUnitId(r.getValue(tsvView.UNIT_ID)) + .withDateTime(r.getValue(tsvView.DATE_TIME)) + .withVersionDate(r.getValue(tsvView.VERSION_DATE)) + .withDataEntryDate(r.getValue(tsvView.DATA_ENTRY_DATE)) + .withValue(r.getValue(tsvView.VALUE)) + .withQualityCode(r.getValue(tsvView.QUALITY_CODE)) + .withStartDate(r.getValue(tsvView.START_DATE)) + .withEndDate(r.getValue(tsvView.END_DATE)) + .build(); + return new RecentValue(tsId, tsv); + }); } From 857a1d7f138f34504e8503ca6d274fb2d78f2a73 Mon Sep 17 00:00:00 2001 From: Rohaan Sandhu Date: Tue, 22 Jul 2025 11:03:23 -0700 Subject: [PATCH 12/19] Add office-id to where condition --- .../main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index fed554bdda..587b99f6a8 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -941,13 +941,18 @@ public List findMostRecentsInRange(String office, List tsId List retval = Collections.emptyList(); if (tsIds != null && !tsIds.isEmpty()) { - String tsFieldName = CWMS_TS_ID; + + // build whereCondition depending on office + Condition whereCondition = AV_CWMS_TS_ID2.CWMS_TS_ID.in(tsIds); + if (office != null) { + whereCondition.and(AV_CWMS_TS_ID2.DB_OFFICE_ID.eq(office)); + } // create baseIds alias CommonTableExpression baseIds = name("base_ids").as( selectDistinct(AV_CWMS_TS_ID2.TS_CODE, AV_CWMS_TS_ID2.CWMS_TS_ID, AV_CWMS_TS_ID2.UNIT_ID) .from(AV_CWMS_TS_ID2) - .where(AV_CWMS_TS_ID2.CWMS_TS_ID.in(tsIds))); + .where(whereCondition)); // convert timestamp to date java.sql.Date startDate = new java.sql.Date(pastdate.getTime()); From d799c99cf175150152e9555927cd0b821cf75a49 Mon Sep 17 00:00:00 2001 From: Rohaan Sandhu Date: Tue, 22 Jul 2025 11:19:27 -0700 Subject: [PATCH 13/19] Add missing whereCondition assignment for office-id --- .../src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index 587b99f6a8..5c50b682d6 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -945,7 +945,7 @@ public List findMostRecentsInRange(String office, List tsId // build whereCondition depending on office Condition whereCondition = AV_CWMS_TS_ID2.CWMS_TS_ID.in(tsIds); if (office != null) { - whereCondition.and(AV_CWMS_TS_ID2.DB_OFFICE_ID.eq(office)); + whereCondition = whereCondition.and(AV_CWMS_TS_ID2.DB_OFFICE_ID.eq(office)); } // create baseIds alias From 9fc4e348a70292f58f08a7203dd3e2a58e10584d Mon Sep 17 00:00:00 2001 From: Rohaan Sandhu Date: Tue, 22 Jul 2025 14:12:24 -0700 Subject: [PATCH 14/19] Fix category and group recent endpoint --- .../src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index 5c50b682d6..8b137cb977 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -1148,7 +1148,7 @@ public List findRecentsInRange(String office, String categoryId, St String tsId = r.getValue(AV_TS_GRP_ASSGN.AV_TS_GRP_ASSGN.TS_ID.getName(), String.class); TsvDqu tsv = new TsvDqu.Builder() .withOfficeId(r.getValue(tsvView.OFFICE_ID)) - .withCwmsTsId(r.getValue(tsvView.CWMS_TS_ID.as(tsId))) + .withCwmsTsId(r.getValue(tsvView.CWMS_TS_ID.as(AV_TS_GRP_ASSGN.AV_TS_GRP_ASSGN.TS_ID.getName()))) .withUnitId(r.getValue(tsvView.UNIT_ID)) .withDateTime(r.getValue(tsvView.DATE_TIME)) .withVersionDate(r.getValue(tsvView.VERSION_DATE)) From e7dc2cc911eaddab23d278b2112c9d3b0ba475ff Mon Sep 17 00:00:00 2001 From: Rohaan Sandhu Date: Wed, 23 Jul 2025 14:23:26 -0700 Subject: [PATCH 15/19] Add conditional for tsv_limited tables union, and Update category/group processing to match new query --- .../cwms/cda/data/dao/TimeSeriesDaoImpl.java | 247 ++++++++++++------ 1 file changed, 162 insertions(+), 85 deletions(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index a3f3fbd91b..52f12a0feb 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -114,6 +114,7 @@ public class TimeSeriesDaoImpl extends JooqDao implements TimeSeries private static final String TS_CODE = "TS_CODE"; private static final String VALUE = "VALUE"; private static final String CWMS_TS_ID = "CWMS_TS_ID"; + private static final String TS_ID = "TS_ID"; private static final String AT_TS_EXTENTS = "AT_TS_EXTENTS"; private static final String VALUE_AT_MAX_DATE = "value_at_max_date"; private static final String CWMS_20 = "CWMS_20"; @@ -1016,7 +1017,9 @@ public List findMostRecentsInRange(String office, List tsId // extract year LocalDate localEndDate = endDate.toLocalDate(); - int year = localEndDate.getYear(); + LocalDate localStartDate = startDate.toLocalDate(); + int year1 = localStartDate.getYear(); + int year2 = localEndDate.getYear(); // helper subquery for SELECT ts_code FROM base_ids @@ -1024,22 +1027,26 @@ public List findMostRecentsInRange(String office, List tsId .from(baseIds); // references to appropriate year tables, current year and past year - Table AT_TSV_PREV_YEAR_TABLE = table(name(CWMS_20, "AT_TSV_" + (year - 1))); - Table AT_TSV_CURR_YEAR_TABLE = table(name(CWMS_20, "AT_TSV_" + year)); + Table AT_TSV_PREV_YEAR_TABLE = table(name(CWMS_20, "AT_TSV_" + year1)); + Table AT_TSV_CURR_YEAR_TABLE = table(name(CWMS_20, "AT_TSV_" + year2)); + + Select prevYearSelect = select(asterisk()) + .from(AT_TSV_PREV_YEAR_TABLE) + .where(field(name(AT_TSV_PREV_YEAR_TABLE.getName(), DATE_TIME), java.sql.Date.class).between(startDate, endDate)) + .and(field(name(AT_TSV_PREV_YEAR_TABLE.getName(), TS_CODE), BigDecimal.class).in(tsCodeSubquery)); + + Select currYearSelect = select(asterisk()) + .from(AT_TSV_CURR_YEAR_TABLE) + .where(field(name(AT_TSV_CURR_YEAR_TABLE.getName(), DATE_TIME), java.sql.Date.class).between(startDate, endDate)) + .and(field(name(AT_TSV_CURR_YEAR_TABLE.getName(), TS_CODE), BigDecimal.class).in(tsCodeSubquery)); + + Select combinedSelect = prevYearSelect; + // union tables if start and end date are not in the same year + if (year1 != year2) { + combinedSelect = prevYearSelect.unionAll(currYearSelect); + } - // create tsvLimited alias - CommonTableExpression tsvLimited = name("tsv_limited").as( - select(asterisk()) - .from(AT_TSV_PREV_YEAR_TABLE) - .where(field(name(AT_TSV_PREV_YEAR_TABLE.getName(), DATE_TIME), java.sql.Date.class).between(startDate, endDate)) - .and(field(name(AT_TSV_PREV_YEAR_TABLE.getName(), TS_CODE), BigDecimal.class).in(tsCodeSubquery)) - .unionAll( - select(asterisk()) - .from(AT_TSV_CURR_YEAR_TABLE) - .where(field(name(AT_TSV_CURR_YEAR_TABLE.getName(), DATE_TIME), java.sql.Date.class).between(startDate, endDate)) - .and(field(name(AT_TSV_CURR_YEAR_TABLE.getName(), TS_CODE), BigDecimal.class).in(tsCodeSubquery)) - ) - ); + CommonTableExpression tsvLimited = name("tsv_limited").as(combinedSelect); // Create table and field references for AT_TS_EXTENTS Table AT_TS_EXTENTS_TABLE = table(name(CWMS_20, AT_TS_EXTENTS)); @@ -1138,84 +1145,154 @@ public List findMostRecentsInRange(String office, List tsId public List findRecentsInRange(String office, String categoryId, String groupId, @NotNull Timestamp pastLimit, @NotNull Timestamp futureLimit, @NotNull UnitSystem unitSystem) { - AV_TSV_DQU tsvView = AV_TSV_DQU.AV_TSV_DQU; // should we look at the daterange and - // possible use 30D view? - Condition whereCondition = - tsvView.VALUE.isNotNull() - .and(tsvView.DATE_TIME.lt(futureLimit)) - .and(tsvView.DATE_TIME.gt(pastLimit)) - .and(tsvView.START_DATE.le(futureLimit)) - .and(tsvView.END_DATE.gt(pastLimit)); - - if (office != null) { - whereCondition = whereCondition.and(AV_TS_GRP_ASSGN.AV_TS_GRP_ASSGN.DB_OFFICE_ID.eq(office)); - } + List retval; + // Create whereCondition for filtering by category, group, and office + Condition whereCondition = DSL.trueCondition(); if (categoryId != null) { whereCondition = whereCondition.and(AV_TS_GRP_ASSGN.AV_TS_GRP_ASSGN.CATEGORY_ID.eq(categoryId)); } - if (groupId != null) { whereCondition = whereCondition.and(AV_TS_GRP_ASSGN.AV_TS_GRP_ASSGN.GROUP_ID.eq(groupId)); } + if (office != null) { + whereCondition = whereCondition.and(AV_TS_GRP_ASSGN.AV_TS_GRP_ASSGN.DB_OFFICE_ID.eq(office)); + } - Field defUnitsField = CWMS_UTIL_PACKAGE.call_GET_DEFAULT_UNITS( - CWMS_TS_PACKAGE.call_GET_BASE_PARAMETER_ID(AV_TSV_DQU.AV_TSV_DQU.TS_CODE), - DSL.val(unitSystem, String.class)) - .as(DEFAULT_UNITS); - Field maxDateTimeField = max(tsvView.DATE_TIME).over(partitionBy(tsvView.TS_CODE)) - .as(MAX_DATE_TIME); - - SelectConditionStep innerSelect - = dsl.select(tsvView.OFFICE_ID, tsvView.TS_CODE, tsvView.DATE_TIME, - tsvView.VERSION_DATE, tsvView.DATA_ENTRY_DATE, tsvView.VALUE, - tsvView.QUALITY_CODE, tsvView.START_DATE, tsvView.END_DATE, tsvView.UNIT_ID, - defUnitsField, - maxDateTimeField, - AV_TS_GRP_ASSGN.AV_TS_GRP_ASSGN.ATTRIBUTE, - AV_TS_GRP_ASSGN.AV_TS_GRP_ASSGN.TS_ID) - .from(tsvView.join(AV_TS_GRP_ASSGN.AV_TS_GRP_ASSGN) - .on(tsvView.TS_CODE.eq(AV_TS_GRP_ASSGN.AV_TS_GRP_ASSGN.TS_CODE.cast(Long.class)))) - .where(whereCondition); - - Field dateTime = innerSelect.field(tsvView.DATE_TIME); - Field unit = innerSelect.field(tsvView.UNIT_ID); - - Field[] queryFields = new Field[]{ - innerSelect.field(tsvView.OFFICE_ID), - innerSelect.field(tsvView.TS_CODE), - innerSelect.field(tsvView.VERSION_DATE), - innerSelect.field(tsvView.DATA_ENTRY_DATE), - innerSelect.field(tsvView.VALUE), - innerSelect.field(tsvView.QUALITY_CODE), - innerSelect.field(tsvView.START_DATE), - innerSelect.field(tsvView.END_DATE), - dateTime, - unit, - innerSelect.field(AV_TS_GRP_ASSGN.AV_TS_GRP_ASSGN.TS_ID), - innerSelect.field(AV_TS_GRP_ASSGN.AV_TS_GRP_ASSGN.ATTRIBUTE)}; - - return dsl.select(queryFields) - .from(innerSelect) - .where(dateTime.eq(maxDateTimeField).and(defUnitsField.eq(unit))) - .orderBy(field(AV_TS_GRP_ASSGN.AV_TS_GRP_ASSGN.ATTRIBUTE.getName())) - .fetch(r -> { - String tsId = r.getValue(AV_TS_GRP_ASSGN.AV_TS_GRP_ASSGN.TS_ID.getName(), String.class); - TsvDqu tsv = new TsvDqu.Builder() - .withOfficeId(r.getValue(tsvView.OFFICE_ID)) - .withCwmsTsId(r.getValue(tsvView.CWMS_TS_ID.as(AV_TS_GRP_ASSGN.AV_TS_GRP_ASSGN.TS_ID.getName()))) - .withUnitId(r.getValue(tsvView.UNIT_ID)) - .withDateTime(r.getValue(tsvView.DATE_TIME)) - .withVersionDate(r.getValue(tsvView.VERSION_DATE)) - .withDataEntryDate(r.getValue(tsvView.DATA_ENTRY_DATE)) - .withValue(r.getValue(tsvView.VALUE)) - .withQualityCode(r.getValue(tsvView.QUALITY_CODE)) - .withStartDate(r.getValue(tsvView.START_DATE)) - .withEndDate(r.getValue(tsvView.END_DATE)) - .build(); - return new RecentValue(tsId, tsv); - }); + CommonTableExpression baseIds = name("base_ids").as( + selectDistinct(AV_TS_GRP_ASSGN.AV_TS_GRP_ASSGN.TS_CODE, AV_TS_GRP_ASSGN.AV_TS_GRP_ASSGN.TS_ID) + .from(AV_TS_GRP_ASSGN.AV_TS_GRP_ASSGN) + .where(whereCondition)); + + // convert timestamp to date + java.sql.Date startDate = new java.sql.Date(pastLimit.getTime()); + java.sql.Date endDate = new java.sql.Date(futureLimit.getTime()); + + // extract year + LocalDate localEndDate = endDate.toLocalDate(); + LocalDate localStartDate = startDate.toLocalDate(); + int year1 = localStartDate.getYear(); + int year2 = localEndDate.getYear(); + + + // helper subquery for SELECT ts_code FROM base_ids + Select> tsCodeSubquery = select(baseIds.field(AV_CWMS_TS_ID2.TS_CODE)) + .from(baseIds); + + // references to appropriate year tables, current year and past year + Table AT_TSV_PREV_YEAR_TABLE = table(name(CWMS_20, "AT_TSV_" + year1)); + Table AT_TSV_CURR_YEAR_TABLE = table(name(CWMS_20, "AT_TSV_" + year2)); + + Select prevYearSelect = select(asterisk()) + .from(AT_TSV_PREV_YEAR_TABLE) + .where(field(name(AT_TSV_PREV_YEAR_TABLE.getName(), DATE_TIME), java.sql.Date.class).between(startDate, endDate)) + .and(field(name(AT_TSV_PREV_YEAR_TABLE.getName(), TS_CODE), BigDecimal.class).in(tsCodeSubquery)); + + Select currYearSelect = select(asterisk()) + .from(AT_TSV_CURR_YEAR_TABLE) + .where(field(name(AT_TSV_CURR_YEAR_TABLE.getName(), DATE_TIME), java.sql.Date.class).between(startDate, endDate)) + .and(field(name(AT_TSV_CURR_YEAR_TABLE.getName(), TS_CODE), BigDecimal.class).in(tsCodeSubquery)); + + Select combinedSelect = prevYearSelect; + // union tables if start and end date are not in the same year + if (year1 != year2) { + combinedSelect = prevYearSelect.unionAll(currYearSelect); + } + + CommonTableExpression tsvLimited = name("tsv_limited").as(combinedSelect); + + // Create table and field references for AT_TS_EXTENTS + Table AT_TS_EXTENTS_TABLE = table(name(CWMS_20, AT_TS_EXTENTS)); + Field AT_TS_EXTENTS_TS_CODE = field(name(CWMS_20, AT_TS_EXTENTS, TS_CODE), BigDecimal.class); + Field AT_TS_EXTENTS_VERSION_TIME = field(name(CWMS_20, AT_TS_EXTENTS, "VERSION_TIME"), java.sql.Date.class); + Field AT_TS_EXTENTS_EARLIEST_ENTRY_TIME = field(name(CWMS_20, AT_TS_EXTENTS, "EARLIEST_ENTRY_TIME"), Timestamp.class); + Field AT_TS_EXTENTS_LATEST_ENTRY_TIME = field(name(CWMS_20, AT_TS_EXTENTS, "LATEST_ENTRY_TIME"), Timestamp.class); + + // Extract repeated TsCode and DateTime + Field tsvLimitedTsCode = field(name(tsvLimited.getName(), TS_CODE), BigDecimal.class); + Field tsvLimitedDateTime = field(name(tsvLimited.getName(), DATE_TIME), java.sql.Date.class); + + // Create max_values alias + CommonTableExpression maxValues = name("max_values").as( + select( + field(name(baseIds.getName(), TS_ID), String.class), + AV_CWMS_TS_ID2.UNIT_ID, + tsvLimitedTsCode, + tsvLimitedDateTime, + field(name(tsvLimited.getName(), VALUE), BigDecimal.class), + field(name(tsvLimited.getName(), VERSION_DATE), java.sql.Date.class), + field(name(tsvLimited.getName(), DATA_ENTRY_DATE), java.sql.Date.class), + field(name(tsvLimited.getName(), QUALITY_CODE), Integer.class), + AT_TS_EXTENTS_EARLIEST_ENTRY_TIME.as(START_DATE), + AT_TS_EXTENTS_LATEST_ENTRY_TIME.as(END_DATE), + max(tsvLimitedDateTime) + .over(partitionBy(tsvLimitedTsCode)) + .as(MAX_DATE_TIME) + ) + .from(tsvLimited) + .join(baseIds).on(field(name(baseIds.getName(), TS_CODE), BigDecimal.class).equal(tsvLimitedTsCode)) + .join(AV_CWMS_TS_ID2).on(field(name(baseIds.getName(), TS_CODE), BigDecimal.class).equal(AV_CWMS_TS_ID2.TS_CODE)) + .join(AT_TS_EXTENTS_TABLE).on( + AT_TS_EXTENTS_TS_CODE.equal(tsvLimitedTsCode) + .and(AT_TS_EXTENTS_VERSION_TIME + .equal(field(name(tsvLimited.getName(), VERSION_DATE), java.sql.Date.class))) + ) + ); + + // Set default units and convert + Field getDefaultUnits = CWMS_UTIL_PACKAGE.call_GET_DEFAULT_UNITS( + CWMS_TS_PACKAGE.call_GET_BASE_PARAMETER_ID(field(name(maxValues.getName(), TS_CODE), BigDecimal.class)), + DSL.val(unitSystem, String.class)); + + Field value = field(name(maxValues.getName(), VALUE), Double.class); + + Field convertUnits = CWMS_UTIL_PACKAGE.call_CONVERT_UNITS( + value, + field(name(maxValues.getName(), UNIT_ID), String.class), + getDefaultUnits + ); + + // Final query + SelectConditionStep> query = dsl.with(baseIds) + .with(tsvLimited) + .with(maxValues) + .select( + field(name(maxValues.getName(), TS_ID), String.class), + field(name(maxValues.getName(), DATE_TIME), java.sql.Date.class), + field(name(maxValues.getName(), VERSION_DATE), java.sql.Date.class), + field(name(maxValues.getName(), DATA_ENTRY_DATE), java.sql.Date.class), + field(name(maxValues.getName(), QUALITY_CODE), Integer.class), + field(name(maxValues.getName(), START_DATE), java.sql.Date.class), + field(name(maxValues.getName(), END_DATE), java.sql.Date.class), + getDefaultUnits.as(DEFAULT_UNITS), + field(name(maxValues.getName(), DATE_TIME), java.sql.Date.class).as(MAX_DATE_TIME), + convertUnits.as(VALUE_AT_MAX_DATE) + ) + .from(maxValues) + .where(field(name(maxValues.getName(), DATE_TIME), java.sql.Date.class) + .eq(field(name(maxValues.getName(), MAX_DATE_TIME), java.sql.Date.class))); + + logger.fine(() -> query.getSQL(ParamType.INLINED)); + + // fetch and build records + retval = query.fetch(r -> { + TsvDqu tsv = new TsvDqu.Builder() + .withOfficeId(office) + .withCwmsTsId(r.getValue(TS_ID, String.class)) + .withUnitId(r.getValue(DEFAULT_UNITS, String.class)) + .withDateTime(r.getValue(DATE_TIME, java.sql.Date.class)) + .withVersionDate(r.getValue(VERSION_DATE, java.sql.Date.class)) + .withDataEntryDate(r.getValue(DATA_ENTRY_DATE, java.sql.Date.class)) + .withValue(r.getValue(VALUE_AT_MAX_DATE, Double.class)) + .withQualityCode(r.getValue(QUALITY_CODE, Long.class)) + .withStartDate(r.getValue(START_DATE, java.sql.Date.class)) + .withEndDate(r.getValue(END_DATE, java.sql.Date.class)) + .build(); + return new RecentValue(r.getValue(TS_ID, String.class), tsv); + }); + + return retval; } From e78cff838b7ffd68d0ddd0d119b4b740c664a21f Mon Sep 17 00:00:00 2001 From: Rohaan Sandhu Date: Wed, 23 Jul 2025 14:44:31 -0700 Subject: [PATCH 16/19] Add select distinct to avoid duplicate rows being returned --- .../src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index 52f12a0feb..70d3727373 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -1257,7 +1257,7 @@ public List findRecentsInRange(String office, String categoryId, St SelectConditionStep> query = dsl.with(baseIds) .with(tsvLimited) .with(maxValues) - .select( + .selectDistinct( field(name(maxValues.getName(), TS_ID), String.class), field(name(maxValues.getName(), DATE_TIME), java.sql.Date.class), field(name(maxValues.getName(), VERSION_DATE), java.sql.Date.class), From 3260771dc25fd84353292b68dcccb4cf506d8205 Mon Sep 17 00:00:00 2001 From: Rohaan Sandhu Date: Wed, 23 Jul 2025 15:40:41 -0700 Subject: [PATCH 17/19] Clean code --- .../cwms/cda/data/dao/TimeSeriesDaoImpl.java | 24 +++++++------------ 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index 70d3727373..f08cfe32fc 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -103,7 +103,7 @@ public class TimeSeriesDaoImpl extends JooqDao implements TimeSeries private static final Logger logger = Logger.getLogger(TimeSeriesDaoImpl.class.getName()); /** - * String constants for accessing alias table columns in FindMostRecentsRange + * String constants for accessing alias tables and columns in TimeSeriesRecent querying */ private static final String DATE_TIME = "DATE_TIME"; private static final String VERSION_DATE = "VERSION_DATE"; @@ -1040,11 +1040,10 @@ public List findMostRecentsInRange(String office, List tsId .where(field(name(AT_TSV_CURR_YEAR_TABLE.getName(), DATE_TIME), java.sql.Date.class).between(startDate, endDate)) .and(field(name(AT_TSV_CURR_YEAR_TABLE.getName(), TS_CODE), BigDecimal.class).in(tsCodeSubquery)); - Select combinedSelect = prevYearSelect; // union tables if start and end date are not in the same year - if (year1 != year2) { - combinedSelect = prevYearSelect.unionAll(currYearSelect); - } + Select combinedSelect = (year1 == year2) + ? prevYearSelect + : prevYearSelect.unionAll(currYearSelect); CommonTableExpression tsvLimited = name("tsv_limited").as(combinedSelect); @@ -1090,10 +1089,8 @@ public List findMostRecentsInRange(String office, List tsId CWMS_TS_PACKAGE.call_GET_BASE_PARAMETER_ID(field(name(maxValues.getName(), TS_CODE), BigDecimal.class)), DSL.val(unitSystem, String.class)); - Field value = field(name(maxValues.getName(), VALUE), Double.class); - Field convertUnits = CWMS_UTIL_PACKAGE.call_CONVERT_UNITS( - value, + field(name(maxValues.getName(), VALUE), Double.class), field(name(maxValues.getName(), UNIT_ID), String.class), getDefaultUnits ); @@ -1194,11 +1191,10 @@ public List findRecentsInRange(String office, String categoryId, St .where(field(name(AT_TSV_CURR_YEAR_TABLE.getName(), DATE_TIME), java.sql.Date.class).between(startDate, endDate)) .and(field(name(AT_TSV_CURR_YEAR_TABLE.getName(), TS_CODE), BigDecimal.class).in(tsCodeSubquery)); - Select combinedSelect = prevYearSelect; // union tables if start and end date are not in the same year - if (year1 != year2) { - combinedSelect = prevYearSelect.unionAll(currYearSelect); - } + Select combinedSelect = (year1 == year2) + ? prevYearSelect + : prevYearSelect.unionAll(currYearSelect); CommonTableExpression tsvLimited = name("tsv_limited").as(combinedSelect); @@ -1245,10 +1241,8 @@ public List findRecentsInRange(String office, String categoryId, St CWMS_TS_PACKAGE.call_GET_BASE_PARAMETER_ID(field(name(maxValues.getName(), TS_CODE), BigDecimal.class)), DSL.val(unitSystem, String.class)); - Field value = field(name(maxValues.getName(), VALUE), Double.class); - Field convertUnits = CWMS_UTIL_PACKAGE.call_CONVERT_UNITS( - value, + field(name(maxValues.getName(), VALUE), Double.class), field(name(maxValues.getName(), UNIT_ID), String.class), getDefaultUnits ); From 43b9b3500aa6349a6d4718e29849dc4dfb38090c Mon Sep 17 00:00:00 2001 From: Rohaan Sandhu Date: Thu, 24 Jul 2025 10:43:32 -0700 Subject: [PATCH 18/19] Add assertion for timing in IT --- .../test/java/cwms/cda/api/TimeSeriesRecentControllerIT.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesRecentControllerIT.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesRecentControllerIT.java index e4c1df2012..b6df9fd64c 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesRecentControllerIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesRecentControllerIT.java @@ -31,6 +31,7 @@ import static cwms.cda.security.ApiKeyIdentityProvider.AUTH_HEADER; import static io.restassured.RestAssured.given; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThan; import static org.junit.jupiter.api.Assertions.assertEquals; import cwms.cda.api.enums.VersionType; @@ -200,6 +201,7 @@ void test_retrieving_recent_ts_data() throws Exception { .assertThat() .statusCode(is(HttpServletResponse.SC_OK)) .body("size()", is(1)) + .time(lessThan(5000L)) ; // get recent data using timeseries id @@ -219,6 +221,7 @@ void test_retrieving_recent_ts_data() throws Exception { .assertThat() .statusCode(is(HttpServletResponse.SC_OK)) .body("size()", is(1)) + .time(lessThan(5000L)) ; } From 6da79da72c82d6710ac23e9995e6e9543d3bcd9f Mon Sep 17 00:00:00 2001 From: Rohaan Sandhu Date: Mon, 28 Jul 2025 11:43:12 -0700 Subject: [PATCH 19/19] Continue using String constants --- .../main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index ff6d12917d..244a0b3c9b 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -341,10 +341,10 @@ protected TimeSeries getRequestedTimeSeries(String page, int pageSize, @NotNull ); // Give the TVQ (time, value, quality) columns names - Field dateTimeCol = field("DATE_TIME", Timestamp.class).as("DATE_TIME"); - Field valueCol = field("VALUE", Double.class).as("VALUE"); - Field qualityCol = field("QUALITY_CODE", Integer.class).as("QUALITY_CODE"); - Field dataEntryDate = field("DATA_ENTRY_DATE", Timestamp.class).as("data_entry_date"); + Field dateTimeCol = field(DATE_TIME, Timestamp.class).as(DATE_TIME); + Field valueCol = field(VALUE, Double.class).as(VALUE); + Field qualityCol = field(QUALITY_CODE, Integer.class).as(QUALITY_CODE); + Field dataEntryDate = field(DATA_ENTRY_DATE, Timestamp.class).as("data_entry_date"); Long beginTimeMilli = beginTime.toInstant().toEpochMilli(); Long endTimeMilli = endTime.toInstant().toEpochMilli();