From 98ffb912f3e3c035808332e7b45308fabcbba686 Mon Sep 17 00:00:00 2001 From: EdmondFrank Date: Wed, 15 Mar 2023 17:01:59 +0800 Subject: [PATCH] Support new trending API (#11) Signed-off-by: EdmondFrank --- app/graphql/types/queries/base_query.rb | 14 +- .../types/queries/community_overview_query.rb | 2 +- app/graphql/types/queries/trending_query.rb | 154 ++++++++++++++++++ app/graphql/types/query_type.rb | 1 + app/graphql/types/trending_type.rb | 13 ++ 5 files changed, 181 insertions(+), 3 deletions(-) create mode 100644 app/graphql/types/queries/trending_query.rb create mode 100644 app/graphql/types/trending_type.rb diff --git a/app/graphql/types/queries/base_query.rb b/app/graphql/types/queries/base_query.rb index 1cbf8321..fed4af8a 100644 --- a/app/graphql/types/queries/base_query.rb +++ b/app/graphql/types/queries/base_query.rb @@ -149,17 +149,27 @@ def normalize_label(label) def extract_repos_count(label, level) if level == 'community' project = ProjectTask.find_by(project_name: label) - project ? director_repo_list(project.remote_url).length : 1 + project ? director_repo_list(project&.remote_url).length : 1 else 1 end end + def extract_name_and_full_path(label) + if label =~ /github\.com\/(.+)\/(.+)/ + [$2, "#{$1}/#{$2}"] + elsif label =~ /gitee\.com\/(.+)\/(.+)/ + [$2, "#{$1}/#{$2}"] + else + [label, label] + end + end + def extract_repos_source(label, level) repo_list = [label] if level == 'community' project = ProjectTask.find_by(project_name: label) - repo_list = director_repo_list(project.remote_url) + repo_list = director_repo_list(project&.remote_url) end github_count, gitee_count = 0,0 repo_list.each do |url| diff --git a/app/graphql/types/queries/community_overview_query.rb b/app/graphql/types/queries/community_overview_query.rb index fcef6418..d125af19 100644 --- a/app/graphql/types/queries/community_overview_query.rb +++ b/app/graphql/types/queries/community_overview_query.rb @@ -19,7 +19,7 @@ def resolve(label: nil, page: 1, per: 9) skeleton = Hash[Types::CommunityOverviewType.fields.keys.zip([])].symbolize_keys result = if project - repo_list = director_repo_list(project.remote_url) + repo_list = director_repo_list(project&.remote_url) current_page = repo_list.in_groups_of(per)&.[]([page.to_i - 1, 0].max) || [] gitee_repos = current_page.select {|row| row =~ /gitee\.com/ } github_repos = current_page.select {|row| row =~ /github\.com/ } diff --git a/app/graphql/types/queries/trending_query.rb b/app/graphql/types/queries/trending_query.rb new file mode 100644 index 00000000..7de7dd06 --- /dev/null +++ b/app/graphql/types/queries/trending_query.rb @@ -0,0 +1,154 @@ +# frozen_string_literal: true + +module Types + module Queries + class TrendingQuery < BaseQuery + TRENDING_CACHE_KEY = 'compass-trending' + + type [Types::TrendingType], null: false + description 'Get trending data of compass' + argument :level, String, required: false, description: 'filter by level (repo/community) default: repo' + + def resolve(level: 'repo') + return [] unless ['repo', 'community'].include?(level) + + # Rails.cache.fetch("#{TRENDING_CACHE_KEY}-#{level}", expires_in: 2.hours) do + top_activity_for_trending = + if level == 'community' + fetch_top_activity_by_phrase(level, nil) + else + fetch_top_activity_by_phrase(level, 'gitee.com') + fetch_top_activity_by_phrase(level, 'github.com') + end + + activity_upward_trending = fetch_top_activity_upward_trending(level) + candidate_set = activity_upward_trending + top_activity_for_trending + candidates_labels = candidate_set.map { |set| set[:label] } + after_filter_labels = filter_by_other_metric_models(candidates_labels, level) + + trendings = [] + candidate_set.each do |set| + if after_filter_labels.include?(set[:label]) + repos_count = extract_repos_count(set[:label], set[:level]) + origin = extract_repos_source(set[:label], set[:level]) + name, full_path = extract_name_and_full_path(set[:label]) + trendings << OpenStruct.new( + { + name: name, + origin: origin, + label: set[:label], + level: set[:level], + full_path: full_path, + activity_score: set[:activity_score], + repos_count: repos_count, + } + ) + end + end + trendings.uniq{ |row| row.label }.sample(10) + # end + end + + def fetch_top_activity_by_phrase(level, domain, limit: 50) + basic = + ActivityMetric + .where(level: level) + .custom(collapse: { field: 'label.keyword' }) + + basic = basic.must(match_phrase: { 'label': domain }) if domain + + basic + .range(:grimoire_creation_date, gte: Date.today.end_of_day - 1.month, lte: Date.today.end_of_day) + .page(1) + .per(limit) + .sort(activity_score: :desc) + .source(['label', 'activity_score']) + .execute + .raw_response['hits']['hits'].map do |row| + { + label: row['_source']['label'], + level: level, + activity_score: row['_source']['activity_score'] + } + end + end + + def fetch_top_activity_upward_trending(level, limit: 50) + ActivityMetric + .where(level: level) + .range(:grimoire_creation_date, gte: Date.today.end_of_day - 1.month, lte: Date.today.end_of_day) + .sort(grimoire_creation_date: :desc) + .per(0) + .aggregate( + { + label_group: { + terms: { + field: "label.keyword", + size: limit + }, + aggs: { + arise: { + date_histogram: { + field: :grimoire_creation_date, + interval: "week", + }, + aggs: { + avg_activity: { + avg: { + field: "activity_score" + } + }, + the_delta: { + derivative: { + buckets_path: "avg_activity" + } + } + } + } + } + } + }) + .execute + .aggregations&.[]('label_group')&.[]('buckets') + .select { |row| row['arise']['buckets'].last&.[]('the_delta')&.[]('value').to_f > 0.001 } + .map do |row| + { + label: row['key'], + level: level, + activity_score: row['arise']['buckets'].last&.[]('avg_activity')&.[]('value').to_f + } || [] + end + end + + def filter_by_other_metric_models(candidate_labels, level) + return candidate_labels if level == 'community' + { community_support_score: CommunityMetric, code_quality_guarantee: CodequalityMetric }.map do |score, metric| + metric + .where(level: level) + .where({'label.keyword' => candidate_labels}) + .range(:grimoire_creation_date, gte: Date.today.end_of_day - 1.month, lte: Date.today.end_of_day) + .per(0) + .aggregate( + { + label_group: { + terms: { + field: "label.keyword", + size: candidate_labels.length + }, + aggs: { + avg_score: { + avg: { + field: score + } + } + } + } + }) + .execute + .aggregations&.[]('label_group')&.[]('buckets') + .select { |row| row['avg_score']&.[]('value').to_f > 0.0 } + .map { |row| row['key'] } + end.reduce(&:&) || [] + end + end + end +end diff --git a/app/graphql/types/query_type.rb b/app/graphql/types/query_type.rb index 79a7bf2a..457b1936 100644 --- a/app/graphql/types/query_type.rb +++ b/app/graphql/types/query_type.rb @@ -11,6 +11,7 @@ class QueryType < Types::BaseObject field :fuzzy_search, resolver: Queries::ProjectFuzzyQuery field :recent_updates, resolver: Queries::ProjectRecentUpdatesQuery field :overview, resolver: Queries::OverviewQuery + field :trending, resolver: Queries::TrendingQuery field :community_overview, resolver: Queries::CommunityOverviewQuery field :bulk_overview, resolver: Queries::BulkOverviewQuery diff --git a/app/graphql/types/trending_type.rb b/app/graphql/types/trending_type.rb new file mode 100644 index 00000000..c212345a --- /dev/null +++ b/app/graphql/types/trending_type.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Types + class TrendingType < Types::BaseObject + field :name, String + field :origin, String + field :label, String + field :level, String + field :full_path, String + field :activity_score, Float + field :repos_count, Float + end +end