diff --git a/.gitignore b/.gitignore index 2663d45d6..72fa48565 100644 --- a/.gitignore +++ b/.gitignore @@ -155,4 +155,7 @@ whitelist.py cloudbuild.yaml # ignore docker related -Docker/.env* \ No newline at end of file +Docker/.env* + +# django admin static assests +static \ No newline at end of file diff --git a/ddpui/admin/templates/static_admin.html b/ddpui/admin/templates/static_admin.html new file mode 100644 index 000000000..12b7601a1 --- /dev/null +++ b/ddpui/admin/templates/static_admin.html @@ -0,0 +1,53 @@ +{% extends "admin/base_site.html" %} + +{% block title %}Long-running flows{% endblock %} + +{% block content %} +

Long-running flows ({{nhours}} hrs)

+{% for flow_run in flow_runs %} +
+

Flow Run

+

State: {{ flow_run.state_name }}

+ {% if flow_run.org_slug %} +

Org Slug: {{ flow_run.org_slug }}

+ {% endif %} + {% if flow_run.tasks %} +

Tasks: {{ flow_run.tasks }}

+ {% endif %} + {% if flow_run.flow_name %} +

Flow Name: {{ flow_run.flow_name }}

+ {% endif %} + {% if flow_run.connection_id %} + {% if flow_run.connection_url %} +

Connection: {{ flow_run.connection_id }}

+ {% else %} +

Connection ID: {{ flow_run.connection_id }}

+ {% endif %} + {% endif %} +
+
+{% endfor %} + +

Airbyte workspaces

+{% for w in workspaces %} +
+

Org: {{w.org_slug}}

+

Workspace: {{ w.workspace_id }}

+
+
+{% endfor %} + + +

LLM feedback

+{% for session in llm_sessions %} +
+

Org: {{session.org_slug}}

+ {% for res in session.response %} +

Prompt: {{res.prompt}}

+

Response: {{res.response}}

+ {% endfor %} +

Feedback: {{session.feedback}}

+
+
+{% endfor %} +{% endblock %} diff --git a/ddpui/admin/views/custom_views.py b/ddpui/admin/views/custom_views.py new file mode 100644 index 000000000..6b93e7392 --- /dev/null +++ b/ddpui/admin/views/custom_views.py @@ -0,0 +1,60 @@ +from django.shortcuts import render +from ddpui.models.tasks import OrgTask +from ddpui.models.org import Org +from ddpui.models.llm import LlmSession +from ddpui.ddpprefect.prefect_service import get_long_running_flow_runs +from ddpui.utils.helpers import find_key_in_dictionary + + +def orgs_index_view(request): + nhours = 2 + flow_runs = get_long_running_flow_runs(nhours) + context = {"flow_runs": [], "nhours": nhours, "workspaces": [], "llm_sessions": []} + + # airbyte workspaces + for org in Org.objects.order_by("name"): + url = f"http://localhost:8000/workspaces/{org.airbyte_workspace_id}" + context["workspaces"].append( + {"org_slug": org.slug, "workspace_url": url, "workspace_id": org.airbyte_workspace_id} + ) + + # llm feedbacks if any + for session in LlmSession.objects.filter(feedback__isnull=False).order_by("-updated_at"): + context["llm_sessions"].append( + { + "org_slug": session.org.slug, + "response": session.response, + "feedback": session.feedback, + "user_prompts": "\n".join(session.user_prompts), + } + ) + + # long running flows + for flow_run in flow_runs: + flow_run_data = { + "state_name": flow_run["state_name"], + "flow_run_url": f"http://localhost:4200/flow-runs/flow-run/{flow_run['id']}", + "org_slug": find_key_in_dictionary(flow_run["parameters"], "org_slug"), + "tasks": [ + x["slug"] for x in find_key_in_dictionary(flow_run["parameters"], "tasks") or [] + ], + "flow_name": find_key_in_dictionary(flow_run["parameters"], "flow_name"), + "connection_id": find_key_in_dictionary(flow_run["parameters"], "connection_id"), + "orgtask_org_slug": None, + "connection_url": None, + } + + connection_id = flow_run_data["connection_id"] + if connection_id: + orgtask = OrgTask.objects.filter(connection_id=connection_id).first() + if orgtask: + flow_run_data["orgtask_org_slug"] = orgtask.org.slug + flow_run_data[ + "connection_url" + ] = f"http://localhost:8000/workspaces/{orgtask.org.airbyte_workspace_id}/connections/{connection_id}" + else: + flow_run_data["orgtask_org_slug"] = connection_id + + context["flow_runs"].append(flow_run_data) + + return render(request, "static_admin.html", context) diff --git a/ddpui/settings.py b/ddpui/settings.py index fc30bbb27..2d3c8b41b 100644 --- a/ddpui/settings.py +++ b/ddpui/settings.py @@ -36,6 +36,12 @@ # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent +STATIC_URL = "/static/" +STATIC_ROOT = os.path.join(BASE_DIR, "static") +# # Additional locations of static files +# STATICFILES_DIRS = [ +# os.path.join(BASE_DIR, "assests"), +# ] # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/4.1/howto/deployment/checklist/ @@ -125,7 +131,7 @@ TEMPLATES = [ { "BACKEND": "django.template.backends.django.DjangoTemplates", - "DIRS": [], + "DIRS": [os.path.join(BASE_DIR, "ddpui", "admin", "templates")], "APP_DIRS": True, "OPTIONS": { "context_processors": [ diff --git a/ddpui/urls.py b/ddpui/urls.py index f5a1a5d97..f950b32fe 100644 --- a/ddpui/urls.py +++ b/ddpui/urls.py @@ -2,6 +2,8 @@ from django.contrib import admin from django.urls import include, path from django.http import HttpResponse +from django.conf import settings +from django.conf.urls.static import static from ddpui.routes import src_api from ddpui.html.docs import get_dbt_docs @@ -11,6 +13,8 @@ from ddpui.websockets.airbyte_consumer import SourceCheckConnectionConsumer from ddpui.websockets.airbyte_consumer import DestinationCheckConnectionConsumer +from ddpui.admin.views.custom_views import orgs_index_view + def trigger_error(request): # pylint: disable=unused-argument # skipcq PYK-W0612 """endpoint to test sentry""" @@ -23,6 +27,7 @@ def healthcheck(request): # pylint:disable=unused-argument urlpatterns = [ + path("admin/orgs/", orgs_index_view, name="orgs_index_view"), path("admin/", admin.site.urls), path("healthcheck", healthcheck), path("docs//", get_dbt_docs), @@ -32,6 +37,8 @@ def healthcheck(request): # pylint:disable=unused-argument path("", src_api.urls), ] +urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) + # socket endpoints ws_urlpatterns = [ path("wss/data_insights/", DataInsightsConsumer.as_asgi()),