From fd6ce238f53a0ff0ab367863cbcacfecf9ecc963 Mon Sep 17 00:00:00 2001 From: Juanma Jurado Date: Mon, 3 Aug 2026 10:33:53 +0200 Subject: [PATCH 1/6] feat(data_drip): self-documenting backfills + searchable catalog (new UI) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-implements the feature from PR #21 on top of the restyled main, using the current design system and conventions instead of the pre-restyle UI. - Add `description` and `instructions` DSL to DataDrip::Backfill, mirroring DataDrip::Script.description. `description` is a one-line catalog summary; `instructions` is Markdown guidance shown in the New Backfill Run form. - Add a searchable, paginated Backfills Catalog at /data_drip/backfills, reachable from a new "Catalog" pill in the shared header. Server-side search (autosubmit) and the shared pagination partial replace the PR's client-side JS. - Render `instructions` server-side via a tiny, dependency-free Markdown renderer (headings, bold, inline code, bullet lists, fenced code blocks) with Tailwind classes, reusing the existing backfill-options fetch β€” dropping the PR's markdown.js and backfill_form_controller.js. - Generalize Paginatable to page an in-memory array (the catalog list). - Document the DSLs + catalog in the README, seed the example backfills, and add the description/instructions hints to the generator template. - Recompile the shipped tailwind.css for the new classes. Specs: full suite green (312 examples). Restyle of #21. Co-Authored-By: Miguel LarraΓ±aga Co-Authored-By: Claude Opus 4.8 --- README.md | 37 ++++- app/assets/stylesheets/data_drip/tailwind.css | 65 +++++++++ .../data_drip/backfill_runs_controller.rb | 4 +- .../data_drip/backfills_controller.rb | 48 +++++++ app/helpers/data_drip/backfill_runs_helper.rb | 128 ++++++++++++++++++ app/helpers/data_drip/backfills_helper.rb | 65 +++++++++ .../data_drip/backfill_runs/new.html.erb | 2 +- app/views/data_drip/backfills/index.html.erb | 75 ++++++++++ app/views/data_drip/shared/_header.html.erb | 9 +- config/routes.rb | 4 + lib/data_drip/backfill.rb | 16 +++ lib/data_drip/concerns/paginatable.rb | 9 +- .../data_drip/templates/backfill.rb.erb | 9 ++ .../backfill_runs_controller_spec.rb | 12 ++ .../data_drip/backfills_controller_spec.rb | 74 ++++++++++ .../data_drip/backfill_runs_helper_spec.rb | 67 +++++++++ .../data_drip/backfills_helper_spec.rb | 63 +++++++++ spec/lib/data_drip/backfill_spec.rb | 31 +++++ spec/lib/data_drip/paginatable_spec.rb | 32 +++++ .../app/backfills/add_birthday_to_employee.rb | 20 +++ .../app/backfills/add_role_to_employee.rb | 15 ++ .../app/backfills/set_employee_role.rb | 2 + 22 files changed, 781 insertions(+), 6 deletions(-) create mode 100644 app/controllers/data_drip/backfills_controller.rb create mode 100644 app/helpers/data_drip/backfills_helper.rb create mode 100644 app/views/data_drip/backfills/index.html.erb create mode 100644 spec/controllers/data_drip/backfills_controller_spec.rb create mode 100644 spec/helpers/data_drip/backfills_helper_spec.rb diff --git a/README.md b/README.md index cdc90a4..b10ea43 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ DataDrip is a Rails engine that provides a robust framework for running data bac - πŸ”§ **Flexible Processing**: Choose between batch-level or element-level processing - πŸ“ˆ **Progress Tracking**: Real-time progress updates and batch monitoring - 🎯 **Scoped Processing**: Define custom scopes for targeted data processing +- πŸ“š **Self-Documenting**: Give backfills a `description` and Markdown `instructions`, browsable in a searchable catalog ## Installation @@ -358,6 +359,38 @@ DataDrip supports various attribute types that automatically generate appropriat - **`:time`** - Time picker - **`:datetime`** - Date and time picker +### Documenting Backfills + +Backfills can document themselves so operators know what each one does and how to fill in its options β€” without reading the source. + +```ruby +class AddRoleToEmployee < DataDrip::Backfill + # One-line summary, listed in the backfills catalog. + description "Assigns the default 'intern' role to employees that don't have one yet." + + # Richer guidance (Markdown), shown in the New Backfill Run form the moment + # this backfill is selected. + instructions <<~MARKDOWN + # Assign default role + Sets the **intern** role on all employees that don't have one yet. + + ## Options + - `age`: Filter employees by age (optional) + - `name`: Filter by exact name match (optional) + MARKDOWN + + attribute :age, :integer + attribute :name, :string + + # ... +end +``` + +- **`description`** β€” a one-line summary shown in the [backfills catalog](#web-interface). Optional; defaults to `nil`. +- **`instructions`** β€” Markdown rendered as formatted rich text in the New Backfill Run form when the backfill is selected. A small, dependency-free renderer supports headings (`#`/`##`/`###`), `**bold**`, `` `inline code` ``, bullet lists, and fenced code blocks. Optional; defaults to `nil`. + +Both are declared with the same setter/getter idiom (a plain `def self.instructions` override also works). + ### Backfill Structure Every backfill must inherit from `DataDrip::Backfill` and implement: @@ -397,7 +430,9 @@ Navigate to `/data_drip/backfill_runs` in your application to access the DataDri - Stop running backfills - Schedule backfills for future execution -When creating a new backfill run, the interface dynamically generates form fields based on the attributes defined in your backfill class, making it easy to customize each run without code changes. +When creating a new backfill run, the interface dynamically generates form fields based on the attributes defined in your backfill class, making it easy to customize each run without code changes. If the backfill declares [`instructions`](#documenting-backfills), they render as formatted guidance above the options. + +You can also browse a searchable **backfills catalog** at `/data_drip/backfills` (the **Catalog** tab in the header): every backfill available in the app, with its description and the configurable fields it accepts β€” search by name, description, or field name to find, say, every backfill that takes `company_ids`. The interface supports light and dark mode (following the OS preference) and ships as precompiled CSS inside the gem β€” host applications need no Node, Tailwind, or any other frontend tooling. diff --git a/app/assets/stylesheets/data_drip/tailwind.css b/app/assets/stylesheets/data_drip/tailwind.css index db1ac52..5a7179b 100644 --- a/app/assets/stylesheets/data_drip/tailwind.css +++ b/app/assets/stylesheets/data_drip/tailwind.css @@ -44,6 +44,7 @@ --color-zinc-800: oklch(27.4% 0.006 286.033); --color-zinc-900: oklch(21% 0.006 285.885); --color-zinc-950: oklch(14.1% 0.005 285.823); + --color-black: #000; --color-white: #fff; --spacing: 0.25rem; --container-lg: 32rem; @@ -69,6 +70,7 @@ --tracking-tight: -0.025em; --tracking-wide: 0.025em; --tracking-widest: 0.1em; + --leading-relaxed: 1.625; --radius-sm: 0.25rem; --radius-md: 0.375rem; --radius-lg: 0.5rem; @@ -368,6 +370,9 @@ .mb-2 { margin-bottom: calc(var(--spacing) * 2); } + .mb-3 { + margin-bottom: calc(var(--spacing) * 3); + } .mb-4 { margin-bottom: calc(var(--spacing) * 4); } @@ -447,6 +452,12 @@ .w-\(--progress\) { width: var(--progress); } + .w-1\/3 { + width: calc(1/3 * 100%); + } + .w-1\/5 { + width: calc(1/5 * 100%); + } .w-9 { width: calc(var(--spacing) * 9); } @@ -468,6 +479,9 @@ .max-w-64 { max-width: calc(var(--spacing) * 64); } + .max-w-80 { + max-width: calc(var(--spacing) * 80); + } .max-w-\[45ch\] { max-width: 45ch; } @@ -537,9 +551,19 @@ .justify-end { justify-content: flex-end; } + .gap-1\.5 { + gap: calc(var(--spacing) * 1.5); + } .gap-3 { gap: calc(var(--spacing) * 3); } + .space-y-0\.5 { + :where(& > :not(:last-child)) { + --tw-space-y-reverse: 0; + margin-block-start: calc(calc(var(--spacing) * 0.5) * var(--tw-space-y-reverse)); + margin-block-end: calc(calc(var(--spacing) * 0.5) * calc(1 - var(--tw-space-y-reverse))); + } + } .gap-x-1 { column-gap: calc(var(--spacing) * 1); } @@ -605,6 +629,9 @@ .overflow-y-auto { overflow-y: auto; } + .rounded { + border-radius: 0.25rem; + } .rounded-full { border-radius: calc(infinity * 1px); } @@ -702,6 +729,9 @@ .bg-zinc-400 { background-color: var(--color-zinc-400); } + .bg-zinc-900 { + background-color: var(--color-zinc-900); + } .bg-zinc-950\/5 { background-color: color-mix(in srgb, oklch(14.1% 0.005 285.823) 5%, transparent); @supports (color: color-mix(in lab, red, red)) { @@ -817,6 +847,9 @@ .pb-3 { padding-bottom: calc(var(--spacing) * 3); } + .pl-5 { + padding-left: calc(var(--spacing) * 5); + } .pl-6 { padding-left: calc(var(--spacing) * 6); } @@ -835,6 +868,9 @@ .align-middle { vertical-align: middle; } + .align-top { + vertical-align: top; + } .font-mono { font-family: var(--font-mono); } @@ -872,6 +908,9 @@ .text-\[0\.6875rem\] { font-size: 0.6875rem; } + .text-\[0\.8125rem\] { + font-size: 0.8125rem; + } .leading-5 { --tw-leading: calc(var(--spacing) * 5); line-height: calc(var(--spacing) * 5); @@ -880,6 +919,10 @@ --tw-leading: calc(var(--spacing) * 6); line-height: calc(var(--spacing) * 6); } + .leading-relaxed { + --tw-leading: var(--leading-relaxed); + line-height: var(--leading-relaxed); + } .font-bold { --tw-font-weight: var(--font-weight-bold); font-weight: var(--font-weight-bold); @@ -953,6 +996,9 @@ .text-white { color: var(--color-white); } + .text-zinc-100 { + color: var(--color-zinc-100); + } .text-zinc-300 { color: var(--color-zinc-300); } @@ -1099,6 +1145,11 @@ color: var(--color-zinc-400); } } + .first\:mt-0 { + &:first-child { + margin-top: calc(var(--spacing) * 0); + } + } .last\:mb-0 { &:last-child { margin-bottom: calc(var(--spacing) * 0); @@ -1322,6 +1373,14 @@ } } } + .dark\:bg-black\/40 { + @media (prefers-color-scheme: dark) { + background-color: color-mix(in srgb, #000 40%, transparent); + @supports (color: color-mix(in lab, red, red)) { + background-color: color-mix(in oklab, var(--color-black) 40%, transparent); + } + } + } .dark\:bg-blue-400\/10 { @media (prefers-color-scheme: dark) { background-color: color-mix(in srgb, oklch(70.7% 0.165 254.624) 10%, transparent); @@ -1715,6 +1774,11 @@ inherits: false; initial-value: 0; } +@property --tw-space-y-reverse { + syntax: "*"; + inherits: false; + initial-value: 0; +} @property --tw-divide-y-reverse { syntax: "*"; inherits: false; @@ -1933,6 +1997,7 @@ --tw-translate-x: 0; --tw-translate-y: 0; --tw-translate-z: 0; + --tw-space-y-reverse: 0; --tw-divide-y-reverse: 0; --tw-border-style: solid; --tw-gradient-position: initial; diff --git a/app/controllers/data_drip/backfill_runs_controller.rb b/app/controllers/data_drip/backfill_runs_controller.rb index db08a7e..5add5a6 100644 --- a/app/controllers/data_drip/backfill_runs_controller.rb +++ b/app/controllers/data_drip/backfill_runs_controller.rb @@ -224,7 +224,9 @@ def backfill_options options: {} ) - html = helpers.backfill_option_inputs(temp_run) + # Instructions + option inputs, so both refresh together when the class + # changes (the backfill-options Stimulus controller swaps this container). + html = helpers.backfill_form_details(temp_run) render json: { html: html } end diff --git a/app/controllers/data_drip/backfills_controller.rb b/app/controllers/data_drip/backfills_controller.rb new file mode 100644 index 0000000..c3a3d0e --- /dev/null +++ b/app/controllers/data_drip/backfills_controller.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module DataDrip + # Catalog of the backfill classes available in the host app. Unlike + # BackfillRunsController (which lists persisted *runs*), this lists the + # backfill *definitions* themselves so users can discover what each one does + # and which options it accepts. + class BackfillsController < DataDrip.base_controller_class.constantize + include DataDrip::Paginatable + include DataDrip::BackfillerContext + + layout "data_drip/layouts/application" + helper DataDrip::BackfillRunsHelper + helper DataDrip::BackfillsHelper + + def index + @query = params[:q].to_s.strip + + # Skip anonymous subclasses (e.g. those created in tests) β€” only real, + # named backfills belong in the catalog. + backfills = + DataDrip.all.select { |klass| klass.name.present? }.sort_by(&:name) + backfills = filter_backfills(backfills, @query) if @query.present? + + pagination_data = paginate_collection(backfills, per_page: 10) + @backfills = pagination_data[:collection] + @pagination = pagination_data + end + + private + + # Client asks for a needle; we match it (case-insensitively) against the + # class name, the description, and each option name β€” so searching + # "company_ids" surfaces every backfill that accepts it. + def filter_backfills(backfills, query) + needle = query.downcase + backfills.select do |klass| + haystack = + [ + klass.name, + (klass.description if klass.respond_to?(:description)), + *klass.backfill_options_class.attribute_types.keys + ].compact.join(" ").downcase + haystack.include?(needle) + end + end + end +end diff --git a/app/helpers/data_drip/backfill_runs_helper.rb b/app/helpers/data_drip/backfill_runs_helper.rb index 6ad62be..8399bdf 100644 --- a/app/helpers/data_drip/backfill_runs_helper.rb +++ b/app/helpers/data_drip/backfill_runs_helper.rb @@ -12,6 +12,21 @@ module BackfillRunsHelper LABEL_CLASSES = "block text-sm font-semibold text-zinc-900 dark:text-white" + # Tailwind classes for the tiny Markdown renderer used by backfill + # instructions. They live in a helper so Tailwind's + # `@source "../../../helpers"` scan compiles them into the shipped build, + # even though the instructions HTML is injected into the page dynamically. + MARKDOWN_STYLES = { + h1: "mt-4 mb-1 text-sm font-semibold text-zinc-900 first:mt-0 dark:text-white", + h2: "mt-4 mb-1 text-xs font-semibold tracking-wide text-drip-700 uppercase first:mt-0 dark:text-drip-300", + h3: "mt-4 mb-1 text-xs font-medium text-zinc-700 first:mt-0 dark:text-zinc-300", + p: "mb-2 text-sm text-pretty text-zinc-600 last:mb-0 dark:text-zinc-300", + ul: "mb-2 list-disc space-y-0.5 pl-5 text-sm text-zinc-600 last:mb-0 dark:text-zinc-300", + pre: "mb-2 overflow-x-auto rounded-lg bg-zinc-900 p-3 font-mono text-xs leading-relaxed text-zinc-100 last:mb-0 dark:bg-black/40", + strong: "font-semibold text-zinc-900 dark:text-white", + code: "rounded bg-zinc-950/5 px-1 py-0.5 font-mono text-[0.8125rem] text-zinc-800 dark:bg-white/10 dark:text-zinc-200" + }.freeze + STATUS_BADGES = { "pending" => { badge: "bg-zinc-50 text-zinc-600 inset-ring-zinc-500/20 " \ @@ -235,6 +250,41 @@ def backfill_option_inputs(backfill_run) ) end + # The dynamic body of the New Backfill Run form for a given run: the + # backfill's instructions (if any) followed by its typed option inputs. + # Rendered server-side both on initial page load and by the + # `backfill-options` controller when the class changes, so both refresh + # together. + def backfill_form_details(backfill_run) + safe_join( + [ + backfill_instructions_block(backfill_run), + backfill_option_inputs(backfill_run) + ] + ) + end + + # Renders a backfill's `instructions` (authored in Markdown) as a styled + # callout shown above the options. Returns "" when the backfill sets none. + def backfill_instructions_block(backfill_run) + backfill_class = backfill_run.backfill_class + return "" unless backfill_class.respond_to?(:instructions) + + instructions = backfill_class.instructions + return "" if instructions.blank? + + content_tag :div, + class: "mb-5 rounded-lg bg-drip-50 p-4 dark:bg-drip-400/10" do + header = + content_tag :h2, + "Instructions", + class: + "mb-3 text-xs font-semibold tracking-wide text-drip-700 " \ + "uppercase dark:text-drip-300" + header + content_tag(:div, render_markdown(instructions)) + end + end + # Renders the typed input fields for an options/inputs schema. Shared by # backfills (prefix `backfill_run[options]`) and scripts (prefix # `script_run[inputs]`) through the `field_prefix` argument. @@ -323,6 +373,84 @@ def typed_option_inputs( private + # A deliberately tiny Markdown-subset renderer for backfill instructions: + # `#`/`##`/`###` headings, `**bold**`, `` `inline code` ``, `- `/`* ` bullet + # lists, and triple-backtick fenced code blocks. Kept dependency-free (this + # is an importmap project with no npm/bundler at runtime); if richer Markdown + # is ever needed, swap in a gem here. All dynamic text is escaped before any + # tag is emitted, so the html_safe result never carries unescaped input. + def render_markdown(text) + html = +"" + in_list = false + in_code = false + code_lines = [] + + text.to_s.split("\n").each do |line| + if line.strip.start_with?("```") + if in_code + html << content_tag(:pre, code_lines.join("\n"), class: MARKDOWN_STYLES[:pre]) + code_lines = [] + in_code = false + else + html << "" if in_list + in_list = false + in_code = true + end + next + end + + if in_code + code_lines << line + next + end + + if line.strip.empty? + html << "" if in_list + in_list = false + next + end + + if (heading = line.match(/\A(\#+)\s+(.+)\z/)) + html << "" if in_list + in_list = false + level = [ heading[1].length, 3 ].min + html << content_tag( + "h#{level + 2}", + render_markdown_inline(heading[2]), + class: MARKDOWN_STYLES[:"h#{level}"] + ) + next + end + + if (bullet = line.match(/\A[-*]\s+(.+)\z/)) + unless in_list + html << %(
    ) + in_list = true + end + html << content_tag(:li, render_markdown_inline(bullet[1])) + next + end + + html << "
" if in_list + in_list = false + html << content_tag(:p, render_markdown_inline(line), class: MARKDOWN_STYLES[:p]) + end + + html << content_tag(:pre, code_lines.join("\n"), class: MARKDOWN_STYLES[:pre]) if in_code + html << "" if in_list + html.html_safe + end + + # Applies inline `**bold**` and `` `code` `` formatting. The text is escaped + # first, so the backreferenced capture is already safe when re-inserted. + def render_markdown_inline(text) + ERB::Util + .html_escape(text) + .gsub(/\*\*(.+?)\*\*/, %(\\1)) + .gsub(/`(.+?)`/, %(\\1)) + .html_safe + end + def build_standard_input(name, type, value, field_prefix, required: false) field_name = "#{field_prefix}[#{name}]" diff --git a/app/helpers/data_drip/backfills_helper.rb b/app/helpers/data_drip/backfills_helper.rb new file mode 100644 index 0000000..b9612ab --- /dev/null +++ b/app/helpers/data_drip/backfills_helper.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +module DataDrip + # View helpers for the backfills catalog (BackfillsController). + module BackfillsHelper + # A backfill's one-line description. Guarded with `respond_to?` so a class + # loaded before the `description` DSL existed (e.g. an older host app that + # hasn't restarted after upgrading) degrades to nil instead of 500ing the + # whole catalog. + def backfill_description(backfill_class) + return unless backfill_class.respond_to?(:description) + + backfill_class.description + end + + # The configurable fields a backfill accepts, as `[{ name:, type: }]`, + # derived from the declared options schema. Introspects + # `backfill_options_class.attribute_types` directly so it renders even for a + # backfill defined before richer introspection existed. + def backfill_configurable_fields(backfill_class) + return [] unless backfill_class.respond_to?(:backfill_options_class) + + backfill_class.backfill_options_class.attribute_types.map do |name, type| + { name: name, type: type.type } + end + end + + # Renders the configurable-field pills for the catalog's last column, or an + # em dash when a backfill takes no options. + def backfill_configurable_field_tags(backfill_class) + fields = backfill_configurable_fields(backfill_class) + if fields.empty? + return content_tag(:span, "β€”", class: "text-zinc-400 dark:text-zinc-600") + end + + content_tag :div, class: "flex flex-wrap gap-1.5" do + safe_join(fields.map { |field| configurable_field_pill(field) }) + end + end + + private + + def configurable_field_pill(field) + content_tag :span, + class: + "inline-flex items-baseline gap-x-1.5 rounded-md bg-zinc-100 " \ + "px-2 py-0.5 dark:bg-white/10" do + safe_join( + [ + content_tag( + :span, + field[:name], + class: "font-mono text-xs text-zinc-700 dark:text-zinc-200" + ), + content_tag( + :span, + field[:type], + class: "font-mono text-[0.6875rem] text-zinc-400 dark:text-zinc-500" + ) + ] + ) + end + end + end +end diff --git a/app/views/data_drip/backfill_runs/new.html.erb b/app/views/data_drip/backfill_runs/new.html.erb index 25f5cfc..f887f3d 100644 --- a/app/views/data_drip/backfill_runs/new.html.erb +++ b/app/views/data_drip/backfill_runs/new.html.erb @@ -81,7 +81,7 @@
- <%= backfill_option_inputs(@run) if @run.backfill_class_name.present? %> + <%= backfill_form_details(@run) if @run.backfill_class_name.present? %>
<% scheduled_later = @run.start_at.present? && @run.start_at > 1.minute.from_now %> diff --git a/app/views/data_drip/backfills/index.html.erb b/app/views/data_drip/backfills/index.html.erb new file mode 100644 index 0000000..fb9b1b7 --- /dev/null +++ b/app/views/data_drip/backfills/index.html.erb @@ -0,0 +1,75 @@ +

Backfills catalog

+

Every backfill available in this app, with what it does and the options it accepts.

+ +<%= turbo_frame_tag "backfills", data: { turbo_action: "advance" } do %> + <%= form_with url: backfills_path, method: :get, + data: { controller: "autosubmit", turbo_action: "advance" }, + class: "mt-6 flex flex-wrap items-center gap-3" do %> +
+ + <%= text_field_tag :q, @query, + placeholder: "Search by name, description, or field…", + autocomplete: "off", + aria: { label: "Search backfills" }, + data: { action: "input->autosubmit#submit" }, + class: "#{DataDrip::BackfillRunsHelper::INPUT_CLASSES} pl-8" %> +
+ <% end %> + + <% if @backfills.any? %> +
+
+ + + + + + + + + + <% @backfills.each do |backfill| %> + + + + + + <% end %> + +
Backfill classDescriptionConfigurable fields
+ <%= backfill.name %> + + <% description = backfill_description(backfill) %> + <% if description.present? %> + <%= description %> + <% else %> + No description + <% end %> + <%= backfill_configurable_field_tags(backfill) %>
+
+
+ + <%= render 'data_drip/shared/pagination', + pagination: @pagination, + item_name: "backfills", + page_param: :page, + additional_params: { q: @query.presence } %> + <% else %> +
+ <% filtered = @query.present? %> +

<%= filtered ? "No backfills match your search." : "No backfills defined yet." %>

+

+ <%= filtered ? + "Try a different name, description, or field name." : + "Define a DataDrip::Backfill subclass to see it listed here." %> +

+ <% if filtered %> +
+ <%= link_to "Clear search", backfills_path, class: secondary_button_classes %> +
+ <% end %> +
+ <% end %> +<% end %> diff --git a/app/views/data_drip/shared/_header.html.erb b/app/views/data_drip/shared/_header.html.erb index 6b5fac5..aa05bc1 100644 --- a/app/views/data_drip/shared/_header.html.erb +++ b/app/views/data_drip/shared/_header.html.erb @@ -8,14 +8,19 @@ "rounded-sm focus-visible:outline-2 focus-visible:outline-offset-2 " \ "focus-visible:outline-drip-700" %> + <% backfills_section = controller_name == "backfill_runs" %> <% scripts_section = controller_name == "script_runs" %> + <% catalog_section = controller_name == "backfills" %>
diff --git a/config/routes.rb b/config/routes.rb index 9b0e296..49598b3 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -17,6 +17,10 @@ to: "backfill_runs#set_timezone", as: :set_timezone_backfill_runs + # Catalog of the backfill *definitions* available in the host app (as opposed + # to backfill_runs, which lists persisted runs). + resources :backfills, only: %i[index] + resources :script_runs, only: %i[index show new create destroy] do get :updates, on: :member get :script_inputs, on: :collection diff --git a/lib/data_drip/backfill.rb b/lib/data_drip/backfill.rb index 226b973..8398ed8 100644 --- a/lib/data_drip/backfill.rb +++ b/lib/data_drip/backfill.rb @@ -24,6 +24,22 @@ def self.backfill_options_class schema_options_class end + # Human-readable one-line summary of what this backfill does, shown in the + # backfills catalog. Acts as both setter (`description "..."`) and getter + # (`description`). Returns nil when unset. Mirrors DataDrip::Script.description. + def self.description(text = nil) + @description = text unless text.nil? + @description + end + + # Markdown guidance shown in the New Backfill Run form when this backfill is + # selected. Same setter/getter idiom as `description` + # (`instructions <<~MARKDOWN ... MARKDOWN`). Returns nil when unset. + def self.instructions(text = nil) + @instructions = text unless text.nil? + @instructions + end + def initialize( batch_size: 100, sleep_time: DataDrip.sleep_time, diff --git a/lib/data_drip/concerns/paginatable.rb b/lib/data_drip/concerns/paginatable.rb index 4632fca..c30bc67 100644 --- a/lib/data_drip/concerns/paginatable.rb +++ b/lib/data_drip/concerns/paginatable.rb @@ -18,7 +18,14 @@ def paginate_collection(collection, per_page: 25, page_param: :page) page = total_pages if total_pages.positive? && page > total_pages offset = (page - 1) * per_page - paginated_collection = collection.limit(per_page).offset(offset) + # Works for both an ActiveRecord relation (the runs lists) and a plain + # Array (the backfills catalog, which paginates an in-memory list). + paginated_collection = + if collection.respond_to?(:limit) + collection.limit(per_page).offset(offset) + else + collection[offset, per_page] || [] + end { collection: paginated_collection, diff --git a/lib/generators/data_drip/templates/backfill.rb.erb b/lib/generators/data_drip/templates/backfill.rb.erb index ab8c125..9d91a8f 100644 --- a/lib/generators/data_drip/templates/backfill.rb.erb +++ b/lib/generators/data_drip/templates/backfill.rb.erb @@ -3,6 +3,15 @@ <% end -%> class <%= class_name %> < DataDrip::Backfill + # Optional: a one-line summary shown in the DataDrip backfills catalog. + # description "Short summary of what this backfill does" + + # Optional: Markdown guidance shown in the New Backfill Run form when this + # backfill is selected. + # instructions <<~MARKDOWN + # Explain what this backfill does and how to fill in each option. + # MARKDOWN + <% if @sorbet_enabled -%> extend T::Sig diff --git a/spec/controllers/data_drip/backfill_runs_controller_spec.rb b/spec/controllers/data_drip/backfill_runs_controller_spec.rb index 2307749..892b77b 100644 --- a/spec/controllers/data_drip/backfill_runs_controller_spec.rb +++ b/spec/controllers/data_drip/backfill_runs_controller_spec.rb @@ -564,6 +564,18 @@ expect(html).to include("backfill_run[options][age]") expect(html).to include("backfill_run[options][name]") end + + it "renders the backfill's instructions (as Markdown) above the options" do + get :backfill_options, + params: { backfill_class_name: "AddRoleToEmployee" } + + html = JSON.parse(response.body)["html"] + expect(html).to include("Instructions") + expect(html).to include("age") # option pill (name) + expect(response.body).to include(">integer") # option pill (type) + end + + it "filters by an option name so operators can find a field" do + get :index, params: { q: "max_age" } + + expect(response.body).to include("SetEmployeeRole") + expect(response.body).not_to include("AddBirthdayToEmployee") + end + + it "lists matching backfills in alphabetical order" do + get :index, params: { q: "employee" } # matches only the three fixtures + + body = response.body + expect(body.index("AddBirthdayToEmployee")).to be < body.index( + "AddRoleToEmployee" + ) + expect(body.index("AddRoleToEmployee")).to be < body.index( + "SetEmployeeRole" + ) + end + + it "shows at most one page (10) of results at a time" do + # More named backfills than fit on a page, all matching one query. + 11.times do |i| + stub_const( + "PageProbe#{format("%02d", i)}Backfill", + Class.new(DataDrip::Backfill) do + def scope + Employee.all + end + + def process_element(_element); end + end + ) + end + + get :index, params: { q: "pageprobe" } + + rows = response.body.scan('") + end + + it "escapes HTML in the source so instructions can't inject markup" do + klass = + Class.new(DataDrip::Backfill) do + instructions " is **bold**" + + def scope + Employee.all + end + + def process_element(_element); end + end + allow(run).to receive(:backfill_class).and_return(klass) + + html = helper.backfill_instructions_block(run) + + expect(html).to include("<script>") + expect(html).not_to include("