Skip to content

Postgres dblinks #85

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Postgres dblinks #85

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2c1b3a9
convert to PG dblinks
steve-fischer-200 Jun 25, 2025
c3b5162
fix REMOTE macros
steve-fischer-200 Jun 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions Model/bin/showNewCommentLinks
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,15 @@ use Data::Dumper;
use Getopt::Long qw(GetOptions);
use ApiCommonData::Load::TuningConfig::Utils;

my ($propfile, $instance, $schema, $password, $suffix, $prefix, $dblink, $subversionDir, $debug);
my ($propfile, $instance, $schema, $password, $suffix, $prefix, $remoteUserSchema, $subversionDir, $debug);
Getopt::Long::Configure("pass_through");
GetOptions("propfile=s" => \$propfile,
"instance=s" => \$instance,
"schema=s" => \$schema,
"password=s" => \$password,
"suffix=s" => \$suffix,
"prefix=s" => \$prefix,
"dblink=s" => \$dblink,
"remoteUserSchema=s" => \$remoteUserSchema,
"subversionDir=s" => \$subversionDir,
"debug!" => \$debug,
);
Expand All @@ -27,7 +27,7 @@ my $dbh = ApiCommonData::Load::TuningConfig::Utils::getDbHandle($instance, $sche
my $csiTable = $prefix . "NewCsi";
my $stmt = $dbh->prepare(<<SQL) or die "preparing new-CommentStableId query";
select nc.comment_stable_id, nc.stable_id, nc.comment_id, c.headline
from $csiTable nc, userlogins5.Comments\@$dblink c
from $csiTable nc, $remoteUserSchema.Comments c
where nc.comment_id = c.comment_id
SQL

Expand Down
3 changes: 1 addition & 2 deletions Model/lib/wdk/apiCommonModel.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,8 +181,7 @@

<exampleStratsAuthor name="VEuPathDB BRC"/>

<declaredMacro name="COMMENT_SCHEMA" usedBy="model" excludeProjects="EuPathDB,UniDB"/>
<declaredMacro name="COMMENT_DBLINK" usedBy="model" excludeProjects="EuPathDB,UniDB"/>
<declaredMacro name="REMOTE_COMMENT_SCHEMA" usedBy="model" excludeProjects="EuPathDB,UniDB"/>
<declaredMacro name="CNDSRC_BIN" usedBy="perl" includeProjects="CryptoDB,PlasmoDB,ToxoDB,TriTrypDB" />
<declaredMacro name="WEBSERVICEMIRROR" usedBy="model"/>
<declaredMacro name="WEBAPP_BASE_URL" usedBy="model"/>
Expand Down
6 changes: 3 additions & 3 deletions Model/lib/wdk/model/questions/params/metricsParams.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@
CASE when project_id in ('EuPathDB') then 'VEuPathDB'
else project_id
end as display
FROM metrics.SEARCHESSTATS@USER_DBLINK@)
FROM @[email protected])
ORDER BY display
]]>
</sql>
Expand All @@ -92,7 +92,7 @@
CASE when project_id in ('EuPathDB') then 'VEuPathDB'
else project_id
end as display
FROM metrics.ORGANISMVIEWSTATS@USER_DBLINK@)
FROM @[email protected])
ORDER BY display
]]>
</sql>
Expand All @@ -108,7 +108,7 @@
CASE when project_id in ('EuPathDB') then 'VEuPathDB'
else project_id
end as display
FROM metrics.GENEPAGETABLESTATS@USER_DBLINK@
FROM @[email protected]
WHERE project_id is not null)
ORDER BY display
]]>
Expand Down
18 changes: 0 additions & 18 deletions Model/lib/wdk/model/questions/params/sharedParams.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1518,8 +1518,6 @@ This parameter allows you to apply the minimum number of peptides to each select
<suggest default="100"/>
</stringParam>

<flatVocabParam name="question_popularity"
queryRef="SharedVQ.QuestionPopularity" />

<!--+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-->
<!-- Strain selection for genes by CNV and genomic segment by ploidy -->
Expand Down Expand Up @@ -2817,22 +2815,6 @@ This parameter allows you to apply the minimum number of peptides to each select
<wsColumn name="term" width="500"/>
</processQuery>


<sqlQuery name="QuestionPopularity" isCacheable="true" doNotTest="true">
<column name="internal"/>
<column name="term"/>
<column name="display"/>
<sql>
SELECT term, display, rownum AS internal
FROM (
SELECT a.question_name AS term, count(s.step_id) AS display
FROM @WDK_ENGINE_SCHEMA@answers@USER_DBLINK@ a, @USER_SCHEMA@steps@USER_DBLINK@ s
WHERE a.answer_id = s.step_id
GROUP BY a.question_name
) t
</sql>
</sqlQuery>

<sqlQuery name="MetadataSpec" doNotTest="1" excludeProjects="EuPathDB">
<paramRef groupRef="paramGroups.dynamicParams" ref="sharedParams.metadata_datasets"/>
<column name="ontology_term_name"/>
Expand Down
2 changes: 1 addition & 1 deletion Model/lib/wdk/model/questions/queries/geneQueries.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6444,7 +6444,7 @@ select distinct ta.gene_source_id
SELECT stable_id, substr(headline, 1, 80) as headline, count(*) as comment_count,
min(comment_date) as min_comment_date,
max(comment_date) as max_comment_date
FROM @COMMENT_SCHEMA@MappedComment@COMMENT_DBLINK@
FROM @REMOTE_COMMENT_SCHEMA@MappedComment
WHERE ('@PROJECT_ID@' = 'UniDB' OR project_name = '@PROJECT_ID@')
and is_visible = 1
GROUP BY stable_id, headline
Expand Down
24 changes: 12 additions & 12 deletions Model/lib/wdk/model/questions/queries/metricsQueries.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
to_char(start_date,'YYYY-MM-DD') as start_date, to_char(end_date,'YYYY-MM-DD') as end_date
FROM (
SELECT project_id as project,unique_visitors,visits,page_data_requests,avg_visit_duration,bandwidth_gb,start_date,end_date
FROM METRICS.AWSTATS@USER_DBLINK@
FROM @[email protected]
WHERE project_id in ( $$projectIdSelect$$ )
AND start_date >= $$date.min$$
AND end_date <= $$date.max$$ +1
Expand Down Expand Up @@ -71,7 +71,7 @@
to_char(start_date,'YYYY-MM-DD') as start_date, to_char(end_date,'YYYY-MM-DD') as end_date
FROM (
SELECT project_id as project, unique_logins, newly_registered, newly_registered_notLogin, total_registered, start_date, end_date
FROM METRICS.LOGINSTATS@USER_DBLINK@
FROM @[email protected]
WHERE project_id in ( $$projectIdSelect$$ )
AND start_date >= $$date.min$$
AND end_date <= $$date.max$$ + 1
Expand Down Expand Up @@ -117,7 +117,7 @@
SELECT project_id as project, registered_users_count, registered_page_count, guest_users_count, guest_page_count,
registered_IP_count, guest_IP_count, guest_users_regPage_pct, guest_IP_regPage_pct, guest_users_regPage_newly_registered_pct,
start_date, end_date
FROM METRICS.PAGEVIEWSTATS@USER_DBLINK@
FROM @[email protected]
WHERE project_id in ( $$projectIdSelect$$ )
AND start_date >= $$date.min$$
AND end_date <= $$date.max$$ + 1
Expand Down Expand Up @@ -152,7 +152,7 @@
url_segment, count, to_char(start_date,'YYYY-MM-DD') as start_date, to_char(end_date,'YYYY-MM-DD') as end_date
FROM (
SELECT project_id as project, url_segment, count, start_date, end_date
FROM METRICS.RECORDPAGEVIEWSTATS@USER_DBLINK@
FROM @[email protected]
WHERE project_id in ( $$projectIdSelect$$ )
AND start_date >= $$date.min$$
AND end_date <= $$date.max$$ + 1
Expand Down Expand Up @@ -192,7 +192,7 @@
to_char(start_date,'YYYY-MM-DD') as start_date, to_char(end_date,'YYYY-MM-DD') as end_date
FROM (
SELECT project_id as project, dataset_type, total_complete, total_incomplete, new_complete, new_incomplete, start_date, end_date
FROM METRICS.USERDATASETSSTATS@USER_DBLINK@
FROM @[email protected]
WHERE project_id in ( $$projectIdNoPortalSelect$$ )
AND start_date >= $$date.min$$
AND end_date <= $$date.max$$ + 1
Expand Down Expand Up @@ -233,15 +233,15 @@
to_char(start_date,'YYYY-MM-DD') as start_date, to_char(end_date,'YYYY-MM-DD') as end_date
FROM (
SELECT ovs.project_id as project, organism, IS_REFERENCE_STRAIN as is_reference, oa.project_id as org_project, count, start_date, end_date
FROM METRICS.ORGANISMVIEWSTATS@USER_DBLINK@ ovs
FROM @[email protected] ovs
LEFT OUTER JOIN apidbtuning.organismattributes oa
ON ovs.organism = oa.ORGANISM_NAME
WHERE ovs.project_id in ( $$projectIdNoAllSelect$$ )
AND start_date >= $$date.min$$
AND end_date <= $$date.max$$ + 1
UNION
SELECT ovs.project_id as project, organism, IS_REFERENCE_STRAIN as is_reference, oa.project_id as org_project, count, start_date, end_date
FROM METRICS.ORGANISMVIEWSTATS@USER_DBLINK@ ovs
FROM @[email protected] ovs
LEFT OUTER JOIN apidbtuning.organismattributes oa
ON ovs.organism = oa.ORGANISM_NAME
WHERE ovs.project_id = 'Unknown'
Expand Down Expand Up @@ -283,7 +283,7 @@
to_char(start_date,'YYYY-MM-DD') as start_date, to_char(end_date,'YYYY-MM-DD') as end_date
FROM (
SELECT project_id as project, number_of_organisms, steps_count, start_date, end_date
FROM METRICS.ORGANISMSPERSTEP@USER_DBLINK@
FROM @[email protected]
WHERE project_id in ( $$projectIdSelect$$ )
AND start_date >= $$date.min$$
AND end_date <= $$date.max$$ + 1
Expand Down Expand Up @@ -320,7 +320,7 @@
to_char(start_date,'YYYY-MM-DD') as start_date, to_char(end_date,'YYYY-MM-DD') as end_date
FROM (
SELECT ovs.project_id as project, ovs.organism, IS_REFERENCE_STRAIN as is_reference, steps_count, oa.project_id as org_project, start_date, end_date
FROM METRICS.STEPSPERORGANISM@USER_DBLINK@ ovs
FROM @[email protected] ovs
LEFT OUTER JOIN apidbtuning.organismattributes oa
ON ovs.organism = oa.ORGANISM_NAME
WHERE ovs.project_id in ( $$projectIdNoAllSelect$$ )
Expand Down Expand Up @@ -360,7 +360,7 @@
to_char(start_date,'YYYY-MM-DD') as start_date, to_char(end_date,'YYYY-MM-DD') as end_date
FROM (
SELECT project_id,RECORD_CLASS_URL_SEGMENT as record_class,table_name,count,start_date,end_date
FROM METRICS.GENEPAGETABLESTATS@USER_DBLINK@
FROM @[email protected]
WHERE start_date >= $$date.min$$
AND end_date <= $$date.max$$ + 1
AND project_id in ( $$projectIdNoPortalSelect$$ )
Expand Down Expand Up @@ -397,7 +397,7 @@
tool,count,to_char(start_date,'YYYY-MM-DD') as start_date, to_char(end_date,'YYYY-MM-DD') as end_date
FROM (
SELECT project_id as project,tool,count,start_date,end_date
FROM METRICS.TOOLSSTATS@USER_DBLINK@
FROM @[email protected]
WHERE project_id in ( $$projectIdSelect$$ )
AND start_date >= $$date.min$$
AND end_date <= $$date.max$$ + 1
Expand Down Expand Up @@ -440,7 +440,7 @@
SELECT project_id as project, steps_count, start_date, end_date,
SUBSTR(search_name, 1, INSTR(search_name, '.')-1) AS record_name,
SUBSTR(search_name, INSTR(search_name, '.')+1) AS singlesearch_name
FROM METRICS.SEARCHESSTATS@USER_DBLINK@
FROM @[email protected]
WHERE project_id in ( $$projectIdSelect$$ )
AND steps_count >= $$step_count.min$$
AND steps_count <= $$step_count.max$$
Expand Down
2 changes: 1 addition & 1 deletion Model/lib/wdk/model/questions/queries/universalQueries.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@
<sql>
<![CDATA[
SELECT userfileid AS source_id, projectname AS project_id
FROM uploads.UserFile@USER_DBLINK@
FROM @[email protected]
WHERE projectname = '@PROJECT_ID@'
]]>
</sql>
Expand Down
54 changes: 27 additions & 27 deletions Model/lib/wdk/model/records/commentTableQueries.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,24 +95,24 @@
&&selectReviewed&&
u.first_name || ' ' || u.last_name || ', ' || u.organization as user_name_org
FROM
@COMMENT_SCHEMA@MappedComment c
@REMOTE_COMMENT_SCHEMA@MappedComment c
INNER JOIN apidbtuning.GeneAttributes ga ON c.project_name = ga.project_id AND c.stable_id = ga.source_id
INNER JOIN @COMMENT_SCHEMA@comment_users u ON c.user_id = u.user_id
INNER JOIN @REMOTE_COMMENT_SCHEMA@comment_users u ON c.user_id = u.user_id
INNER JOIN apidbtuning.annotatedgenomesummary ags ON ga.taxon_id = ags.taxon_id
LEFT JOIN (
SELECT comment_id, count(*) as filecount
FROM @COMMENT_SCHEMA@CommentFile
FROM @REMOTE_COMMENT_SCHEMA@CommentFile
GROUP BY comment_id
) files ON c.comment_id = files.comment_id
LEFT JOIN (
SELECT comment_id, string_agg(source_id,',') as pmids
FROM @COMMENT_SCHEMA@CommentReference
FROM @REMOTE_COMMENT_SCHEMA@CommentReference
WHERE database_name='pubmed'
GROUP BY comment_id
) refs ON c.comment_id = refs.comment_id
LEFT JOIN (
SELECT comment_id, COUNT(DISTINCT stable_id) AS geneCount
FROM @COMMENT_SCHEMA@MappedComment
FROM @REMOTE_COMMENT_SCHEMA@MappedComment
WHERE stable_id IN (SELECT source_id FROM apidbTuning.GeneAttributes)
GROUP BY comment_id
) gene_counts ON c.comment_id = gene_counts.comment_id
Expand Down Expand Up @@ -194,28 +194,28 @@
u.first_name || ' ' || u.last_name || ', ' || u.organization as user_name_org,
authortable.authors
FROM
@COMMENT_SCHEMA@MappedComment c
INNER JOIN @COMMENT_SCHEMA@comment_users u c.user_id = u.user_id
@REMOTE_COMMENT_SCHEMA@MappedComment c
INNER JOIN @REMOTE_COMMENT_SCHEMA@comment_users u c.user_id = u.user_id
LEFT JOIN (
SELECT comment_id, count(*) as filecount
FROM @COMMENT_SCHEMA@CommentFile
FROM @REMOTE_COMMENT_SCHEMA@CommentFile
GROUP BY comment_id
) files ON c.comment_id = files.comment_id
LEFT JOIN (
SELECT comment_id, string_agg(source_id,',') as pmids
FROM @COMMENT_SCHEMA@CommentReference
FROM @REMOTE_COMMENT_SCHEMA@CommentReference
WHERE database_name='pubmed'
GROUP BY comment_id
) refs ON c.comment_id = refs.comment_id
LEFT JOIN (
SELECT comment_id, string_agg(source_id,',') as authors
FROM @COMMENT_SCHEMA@CommentReference
FROM @REMOTE_COMMENT_SCHEMA@CommentReference
WHERE database_name='author'
GROUP BY comment_id
) authortable ON c.comment_id = authortable.comment_id
LEFT JOIN (
SELECT comment_id, count(stable_id) as geneCount
FROM @COMMENT_SCHEMA@MappedComment
FROM @REMOTE_COMMENT_SCHEMA@MappedComment
GROUP BY comment_id
) gene_counts c.comment_id = gene_counts.comment_id
WHERE ('@PROJECT_ID@' = 'UniDB' OR c.project_name = '@PROJECT_ID@')
Expand Down Expand Up @@ -252,22 +252,22 @@
SELECT c.comment_target_id, c.project_name as project_id, c.comment_id, c.stable_id, c.comment_date, c.headline,
files.filecount, refs.pmids, gene_counts.geneCount,
u.first_name || ' ' || u.last_name || ', ' || u.organization as user_name_org
FROM @COMMENT_SCHEMA@MappedComment c
INNER JOIN @COMMENT_SCHEMA@comment_users u ON c.user_id = u.user_id
FROM @REMOTE_COMMENT_SCHEMA@MappedComment c
INNER JOIN @REMOTE_COMMENT_SCHEMA@comment_users u ON c.user_id = u.user_id
LEFT JOIN (
SELECT comment_id, count(*) as filecount
FROM @COMMENT_SCHEMA@CommentFile
FROM @REMOTE_COMMENT_SCHEMA@CommentFile
GROUP BY comment_id
) files ON c.comment_id = files.comment_id
LEFT JOIN (
SELECT comment_id, string_agg(source_id,',') as pmids
FROM @COMMENT_SCHEMA@CommentReference
FROM @REMOTE_COMMENT_SCHEMA@CommentReference
WHERE database_name='pubmed'
GROUP BY comment_id
) refs ON c.comment_id = refs.comment_id
LEFT JOIN (
SELECT comment_id, count(stable_id) as geneCount
FROM @COMMENT_SCHEMA@mappedComment
FROM @REMOTE_COMMENT_SCHEMA@mappedComment
GROUP BY comment_id
) gene_counts ON c.comment_id = gene_counts.comment_id
WHERE ('@PROJECT_ID@' = 'UniDB' OR c.project_name = '@PROJECT_ID@')
Expand Down Expand Up @@ -297,9 +297,9 @@
<![CDATA[
SELECT c.stable_id AS source_id, c.project_name as project_id,
lower(t.category) || ':' || count(t.category) as category_link
FROM @COMMENT_SCHEMA@MappedCcomment c,
@COMMENT_SCHEMA@TargetCategory t,
@COMMENT_SCHEMA@CommentTargetCategory tc
FROM @REMOTE_COMMENT_SCHEMA@MappedCcomment c,
@REMOTE_COMMENT_SCHEMA@TargetCategory t,
@REMOTE_COMMENT_SCHEMA@CommentTargetCategory tc
WHERE c.comment_id = tc.comment_id
AND t.target_category_id = tc.target_category_id
GROUP BY c.stable_id, t.category
Expand Down Expand Up @@ -338,22 +338,22 @@
SELECT c.comment_target_id, c.project_name as project_id, c.comment_id, c.stable_id, c.comment_date, c.headline,
files.filecount, refs.pmids, gene_counts.geneCount,
u.first_name || ' ' || u.last_name || ', ' || u.organization as user_name_org
FROM @COMMENT_SCHEMA@MappedComment c
INNER JOIN @COMMENT_SCHEMA@comment_users u ON c.user_id = u.user_id
FROM @REMOTE_COMMENT_SCHEMA@MappedComment c
INNER JOIN @REMOTE_COMMENT_SCHEMA@comment_users u ON c.user_id = u.user_id
LEFT JOIN (
SELECT comment_id, count(*) as filecount
FROM @COMMENT_SCHEMA@CommentFile
FROM @REMOTE_COMMENT_SCHEMA@CommentFile
GROUP BY comment_id
) files ON c.comment_id = files.comment_id
LEFT JOIN (
SELECT comment_id, string_agg(source_id,',') as pmids
FROM @COMMENT_SCHEMA@CommentReference
FROM @REMOTE_COMMENT_SCHEMA@CommentReference
WHERE database_name='pubmed'
GROUP BY comment_id
) refs ON c.comment_id = refs.comment_id
LEFT JOIN (
SELECT comment_id, count(stable_id) as geneCount
FROM @COMMENT_SCHEMA@MappedComment
FROM @REMOTE_COMMENT_SCHEMA@MappedComment
GROUP BY comment_id
) gene_counts ON c.comment_id = gene_counts.comment_id
WHERE ('@PROJECT_ID@' = 'UniDB' OR c.project_name = '@PROJECT_ID@')
Expand Down Expand Up @@ -392,13 +392,13 @@
c.comment_date, c.headline,
files.filecount, references.pmids,
u.first_name || ' ' || u.last_name || ', ' || u.organization as user_name_org
FROM @COMMENT_SCHEMA@MappedComment c, @COMMENT_SCHEMA@comment_users u,
FROM @REMOTE_COMMENT_SCHEMA@MappedComment c, @REMOTE_COMMENT_SCHEMA@comment_users u,
(SELECT comment_id, count(*) as filecount
FROM @COMMENT_SCHEMA@CommentFile
FROM @REMOTE_COMMENT_SCHEMA@CommentFile
GROUP BY comment_id) files,
(SELECT comment_id,
string_agg(source_id,',') as pmids
FROM @COMMENT_SCHEMA@CommentReference
FROM @REMOTE_COMMENT_SCHEMA@CommentReference
WHERE database_name='pubmed'
GROUP BY comment_id
) references
Expand Down
Loading