Skip to content

Special processing for gene orthologs table when downloading result #130

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Special processing for gene orthologs table when downloading result #130

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 13 additions & 7 deletions Model/src/main/java/org/gusdb/wdk/model/answer/AnswerValue.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -366,23 +366,29 @@ public String getAnswerTableSql(Query tableQuery)
getPagedTableSql(tableQuery);
}

public ResultList getTableFieldResultList(TableField tableField) throws WdkModelException {

public String getTableFieldResultSql(TableField tableField) throws WdkModelException {
// has to get a clean copy of the attribute query, without pk params appended
Query tableQuery = tableField.getUnwrappedQuery();

// get and run the paged table query sql
LOG.debug("AnswerValue: getTableFieldResultList(): going to getPagedTableSql()");
LOG.debug("AnswerValue: getTableFieldResultSql(): going to getPagedTableSql()");

return getAnswerTableSql(tableQuery);
}

public ResultList getTableFieldResultList(TableField tableField) throws WdkModelException {
return getTableFieldResultList(tableField, getTableFieldResultSql(tableField));
}

public ResultList getTableFieldResultList(TableField tableField, String customSql) throws WdkModelException {

String sql = getAnswerTableSql(tableQuery);

LOG.debug("AnswerValue: getTableFieldResultList(): back from getPagedTableSql()");
DatabaseInstance platform = _wdkModel.getAppDb();
DataSource dataSource = platform.getDataSource();
ResultSet resultSet = null;
try {
LOG.debug("AnswerValue: getTableFieldResultList(): returning SQL for TableField '" + tableField.getName() + "': \n" + sql);
resultSet = SqlUtils.executeQuery(dataSource, sql, tableQuery.getFullName() + "_table");
LOG.debug("AnswerValue: getTableFieldResultList(): returning SQL for TableField '" + tableField.getName() + "': \n" + customSql);
resultSet = SqlUtils.executeQuery(dataSource, customSql, tableField.getUnwrappedQuery().getFullName() + "_table");
}
catch (SQLException e) {
throw new WdkModelException(e);
Expand Down
14 changes: 14 additions & 0 deletions Model/src/main/java/org/gusdb/wdk/model/answer/stream/SingleTableRecordStream.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
import java.util.Map;
import java.util.NoSuchElementException;

import org.apache.log4j.Logger;
import org.gusdb.fgputil.Timer;
import org.gusdb.fgputil.Wrapper;
import org.gusdb.fgputil.iterator.ReadOnlyIterator;
import org.gusdb.wdk.model.WdkModelException;
Expand All @@ -23,6 +25,8 @@

public class SingleTableRecordStream implements RecordStream {

private static final Logger LOG = Logger.getLogger(SingleTableRecordStream.class);

private AnswerValue _answerValue;
private TableField _tableField;
private PrimaryKeyDefinition _pkDef;
Expand All @@ -34,6 +38,10 @@ public SingleTableRecordStream(AnswerValue answerValue, TableField tableField) t
init(answerValue, tableField, answerValue.getTableFieldResultList(tableField));
}

public SingleTableRecordStream(AnswerValue answerValue, TableField tableField, String customTableSql) throws WdkModelException {
init(answerValue, tableField, answerValue.getTableFieldResultList(tableField, customTableSql));
}

public SingleTableRecordStream(AnswerValue answerValue, TableField tableField, ResultList resultList) {
init(answerValue, tableField, resultList);
}
Expand Down Expand Up @@ -84,6 +92,8 @@ public RecordInstance next() {
_lastPkValues.set(null); // will reset if there is another record after this one

// loop through the ResultList's rows and add to table until PK values differ
int rowCount = 0;
Timer t = new Timer();
while (_resultList.next()) {
Map<String,Object> rowPkValues = new PrimaryKeyValue(_pkDef, _resultList).getRawValues();
if (rawValuesDiffer(currentRecordPkValues, rowPkValues)) {
Expand All @@ -92,6 +102,10 @@ public RecordInstance next() {
return record;
}
// otherwise add row to table value
rowCount++;
LOG.trace("Row " + rowCount + ": fetched in " + t.getElapsedStringAndRestart());
if (rowCount > TableValue.MAX_TABLE_VALUE_ROWS)
throw new WdkRuntimeException("Table query returned too many (>" + TableValue.MAX_TABLE_VALUE_ROWS + ") rows.");
tableValue.initializeRow(_resultList);
}

Expand Down
6 changes: 6 additions & 0 deletions Model/src/main/java/org/gusdb/wdk/model/record/DynamicTableValue.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import java.util.Iterator;

import org.apache.log4j.Logger;
import org.gusdb.fgputil.Timer;
import org.gusdb.wdk.model.WdkModelException;
import org.gusdb.wdk.model.WdkRuntimeException;
import org.gusdb.wdk.model.answer.AnswerValue;
Expand Down Expand Up @@ -36,7 +37,12 @@ public DynamicTableValue(PrimaryKeyValue primaryKey, TableField tableField, User

private void loadRowsFromQuery() {
try (ResultList resultList = _queryInstance.getResults()) {
int rowCount = 0;
Timer t = new Timer();
while (resultList.next()) {
LOG.trace("Row " + (++rowCount) + ": fetched in " + t.getElapsedStringAndRestart());
if (rowCount > MAX_TABLE_VALUE_ROWS)
throw new WdkRuntimeException("Table query returned too many (>" + MAX_TABLE_VALUE_ROWS + ") rows.");
initializeRow(resultList);
}
}
Expand Down
2 changes: 2 additions & 0 deletions Model/src/main/java/org/gusdb/wdk/model/record/TableValue.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@
*/
public class TableValue implements Iterable<TableValueRow> {

public static final int MAX_TABLE_VALUE_ROWS = 30000;

protected final TableField _tableField;
protected List<TableValueRow> _rows;

Expand Down
13 changes: 8 additions & 5 deletions Model/src/main/java/org/gusdb/wdk/model/report/reporter/TableTabularReporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,8 @@

import org.gusdb.wdk.model.WdkModelException;
import org.gusdb.wdk.model.WdkUserException;
import org.gusdb.wdk.model.answer.single.SingleRecordAnswerValue;
import org.gusdb.wdk.model.answer.stream.FileBasedRecordStream;
import org.gusdb.wdk.model.answer.stream.RecordStream;
import org.gusdb.wdk.model.answer.stream.SingleRecordStream;
import org.gusdb.wdk.model.answer.stream.SingleTableRecordStream;
import org.gusdb.wdk.model.record.RecordClass;
import org.gusdb.wdk.model.record.RecordInstance;
Expand All @@ -26,7 +24,7 @@

public class TableTabularReporter extends AbstractTabularReporter {

private TableField _tableField;
protected TableField _tableField;
private Collection<AttributeField> _tableAttributes;

@Override
Expand Down Expand Up @@ -58,14 +56,15 @@ private TableTabularReporter setTable() throws ReporterConfigException {
*/
@Override
public RecordStream getRecords() throws WdkModelException {
/* Test to see if this case is necessary- may be ok to fall down to SingleTableRecordStream
if (_baseAnswer instanceof SingleRecordAnswerValue) {
try {
return new SingleRecordStream((SingleRecordAnswerValue)_baseAnswer);
}
catch (WdkUserException e) {
throw new WdkModelException(e.getMessage(), e);
}
}
} */
RecordClass recordClass = _baseAnswer.getQuestion().getRecordClass();
if (idAttributeContainsNonPkFields(recordClass)) {
// need to use FileBasedRecordStream to support both this table and any needed attributes
Expand All @@ -75,10 +74,14 @@ public RecordStream getRecords() throws WdkModelException {
}
else {
// the records returned by this stream will have only PK and this single table field populated
return new SingleTableRecordStream(_baseAnswer, _tableField);
return getSingleTableRecordStream();
}
}

protected RecordStream getSingleTableRecordStream() throws WdkModelException {
return new SingleTableRecordStream(_baseAnswer, _tableField);
}

private static boolean idAttributeContainsNonPkFields(RecordClass recordClass) throws WdkModelException {
Collection<AttributeField> deps = recordClass.getIdAttributeField().getDependencies();
List<String> pkColumns = Arrays.asList(recordClass.getPrimaryKeyDefinition().getColumnRefs());
Expand Down