[Framework] New "milestones" column in summary tables (#267)

Relates to #260

The new `milestones` column type renders known publication milestones for the
underlying specification. Milestones are retrieved from the W3C Spec dashboard:
https://github.com/w3c/spec-dashboard/tree/gh-pages/pergroup
(files whose name ends with `-milestones.json`).

That columns should typically be added to the summary tables at the end of
"Technologies in progress" sections. This is not done by default for now,
because the milestone information available is still too scarce. Hopefully,
this will change in the near future.

README.md

@@ -66,6 +66,7 @@ Depending on the advancement of the underlying specification, the JSON object ca
 * `publisher`: the organization that published the specification. The framework automatically computes the publisher for W3C, WHATWG, and IETF specifications.
 * `informative`: when the specification is unknown to the [W3C API](https://w3c.github.io/w3c-api/), set the `informative` property to `true` to tell the framework that it only contains informative content or that it will be (or has been) published as a Group Note and not as a Recommendation.
 * `evergreen`: from time to time, specifications remain as drafts indefinitely but are continuously updated and can be considered stable whenever a new version is published. Set the `evergreen` property to `true` when the specification can always be used as a reference, no matter where it is on the Recommendation track.
+* `milestones`: When the [Milestone tracker](https://github.com/w3c/spec-dashboard/#milestone-tracker) does not know anything about the specification, you may set the `milestones` properties to planned publication milestones. Value must be an object whose keys are the planned maturity level (e.g. `CR`, `REC`) and whose values are the planned publication date under the form `YYYY-MM-DD`. Do not use that property for W3C specs as milestones should rather be entered in the milestone tracker!
 * `seeAlso`: a list of other resources that could be worth looking at in relation with the specification. The `seeAlso` property should be an array of objects that have a `url` property set to the URL of the resource, a `label` property set to the title of the resource, and optionally a `kind` property that specifies the kind of resource as a string. The links are rendered in the "See also" column. The whole list is rendered by default, the `kind` value can be used to filter resources in some cases. See [Customizing summary tables](#customizing-summary-tables) for details.
 
 Here is an example of a JSON file that describes the "Intersection Observer" specification:
@@ -335,6 +336,7 @@ The framework recognizes the following column types:
 - `spec` - Specification / Group: Renders the spec title and the name of the group that develops it.
 - `maturity` - Maturity: Renders the maturity status of a spec as an icon. The list of icons is e.g. described in the [About page of the mobile roadmap](https://w3c.github.io/web-roadmaps/mobile/about.html#maturity-levels).
 - `impl` - Implementation status: Renders the implementation status of the specification in main browsers. The icons and info that get represented are e.g. described in the [About page of the mobile rodmap](https://w3c.github.io/web-roadmaps/mobile/about.html#implementation)
+- `milestones` - Milestones: Renders planned publication milestones for the specification. The information is extracted from the [Milestone tracker](https://github.com/w3c/spec-dashboard/#milestone-tracker) project, or from the specification's description. Beware though, as of June 2018, the tracker does not have nearly enough data for this column to be useful.
 - `seeAlso` - See also: Renders the list of related resources, including a link to the Editor's Draft, and a link to the repository. The exact kinds of resources to render can be specified in a `kinds` property. Default value is `all` to render all links, but the property can be set to an array of strings. Possible string values are:
   - `edDraft`: renders a link to the Editor's Draft, when known
   - `repository`: renders a link to the repository that contains the Editor's Draft, when known
@@ -421,7 +423,8 @@ The `js/translations.xx.json` file, where `xx` is the BCP47 language code, needs
     "maturity": "",
     "impl": "",
     "implintents": "",
-    "versions": ""
+    "seeAlso": "",
+    "milestones": ""
   },
   "implstatus": {
     "shipped": "",
@@ -447,6 +450,17 @@ The `js/translations.xx.json` file, where `xx` is the BCP47 language code, needs
     "audio element": "",
     "picture element": "",
     "...": ""
+  },
+  "maturity": {
+    "ED": "",
+    "WD": "",
+    "LS": "",
+    "CR": "",
+    "PR": "",
+    "REC": "",
+    "Retired": "",
+    "NOTE": "",
+    "REF": ""
   }
 }
 ```

assets/css/theme.css

@@ -155,6 +155,10 @@ td ul {
     padding-left: 0;
     margin-top: -8px;
 }
+td.milestones {
+  font-size: 90%;
+  min-width: 150px;
+}
 
 
 /************************************************************

js/generate-utils.js

@@ -135,6 +135,10 @@ const maturityLevels = {
  * Lists of columns in generated tables per type of table
  *
  * This structure may be completed or overridden in `toc.json` files.
+ *
+ * TODO: Add "milestones" to "in-progress" table, once we have enough data. As
+ * of June 2018, the info from spec dashboard is too scarce:
+ * https://github.com/w3c/spec-dashboard/tree/gh-pages/pergroup
  */
 const tableColumnsPerType = {
   'well-deployed': ['feature', 'spec', 'maturity', 'impl'],
@@ -376,13 +380,55 @@ const createSeeAlsoCell = function (column, featureId, featureName, specInfo, im
   return cell;
 };
 
+const createMilestonesCell = function (column, featureId, featureName, specInfo, implInfo, translate, lang, pos) {
+  let cell = document.createElement('td');
+  cell.classList.add('milestones');
+  if (specInfo.milestones) {
+    let milestones = Object.keys(specInfo.milestones).map(maturity => {
+      return {
+        date: specInfo.milestones[maturity],
+        maturity
+      }
+    }).sort((a, b) => {
+      if (a.date < b.date) {
+        return -1;
+      }
+      else if (a.date > b.date) {
+        return 1;
+      }
+      return 0;
+    });
+
+    milestones.forEach((milestone, pos) => {
+      if (pos > 0) {
+        cell.appendChild(document.createElement('br'));
+      }
+      let label = translate('maturity', milestone.maturity);
+      if (label !== milestone.maturity) {
+        let el = document.createElement('abbr');
+        el.setAttribute('title', label);
+        el.appendChild(document.createTextNode(milestone.maturity));
+        cell.appendChild(el);
+      }
+      else {
+        cell.appendChild(document.createTextNode(milestone.maturity));
+      }
+      cell.appendChild(document.createTextNode(
+        ': ' + formatMonthAndYearDate(new Date(milestone.date), lang)));
+    });
+  }
+  return cell;
+};
+
+
 const tableColumnCreators = {
   'feature': createFeatureCell,
   'spec': createSpecCell,
   'maturity': createMaturityCell,
   'impl': createImplCell,
   'impl-intents': createImplCell,
-  'seeAlso': createSeeAlsoCell
+  'seeAlso': createSeeAlsoCell,
+  'milestones': createMilestonesCell
 };
 
 

js/translations.json

@@ -12,7 +12,8 @@
     "maturity": "Maturity",
     "impl": "Current implementations",
     "impl-intents": "Implementation intents",
-    "seeAlso": "See also"
+    "seeAlso": "See also",
+    "milestones": "Milestones"
   },
   "implstatus": {
     "shipped": "Shipped",

tools/extract-spec-data.js

@@ -220,7 +220,7 @@ function getUrlForReverseLookup(spec) {
 async function fetchJson(url, options) {
   let response = await fetch(url, options);
   if (response.status !== 200) {
-    throw new Error(`Fetch returned a non OK HTTP status code (url: ${url})`);
+    throw new Error(`Fetch returned a non OK HTTP status code (url: ${url}, status: ${response.status})`);
   }
   return response.json();
 }
@@ -255,6 +255,11 @@ async function extractSpecData(files, config) {
     }
   };
 
+  let githubHttpOptions = {
+    agent: new https.Agent({ maxSockets: 2 }),
+    keepAlive: true
+  };
+
   // Fetch spec info from Specref when spec is not a TR spec.
   // (Proceed in chunks not to end up with a URL that is thousands of bytes
   // long, and only fetch a given lookup URL once)
@@ -276,6 +281,24 @@ async function extractSpecData(files, config) {
     specsInfo = Object.assign(specsInfo, info);
   }
 
+  // In-memory cache for milestones per group
+  // (used to avoid fetching the milestones more than once)
+  let deliverersMilestones = {};
+  async function fetchMilestones(deliverer) {
+    if (!deliverersMilestones[deliverer.id]) {
+      deliverersMilestones[deliverer.id] = fetchJson(
+        `https://w3c.github.io/spec-dashboard/pergroup/${deliverer.id}-milestones.json`,
+        githubHttpOptions
+      ).catch(err => {
+        console.warn(`- ${deliverer.name} (id: ${deliverer.id}): Could not retrieve milestones file`);
+        return null;
+      });
+    }
+
+    let milestonesJson = await deliverersMilestones[deliverer.id];
+    return milestonesJson;
+  }
+
   async function fetchSpecInfo(spec) {
     let trInfo = {};
     let lookupInfo = {};
@@ -302,6 +325,14 @@ async function extractSpecData(files, config) {
         label: deliverer.name,
         url: deliverer._links.homepage.href
       }));
+
+      // Retrieve milestones info from dashboard repo
+      for (deliverer of deliverersJson._embedded.deliverers) {
+        let milestones = await fetchMilestones(deliverer);
+        if (milestones && milestones[latestInfo.shortlink]) {
+          trInfo.milestones = milestones[latestInfo.shortlink];
+        }
+      }
     }
     else {
       // For other specs, use info returned by Specref
@@ -316,7 +347,8 @@ async function extractSpecData(files, config) {
       status: spec.data.status || trInfo.status || lookupInfo.status || 'ED',
       deliveredBy: spec.data.wgs || trInfo.deliveredBy || lookupInfo.deliveredBy || [],
       publisher: spec.data.publisher || trInfo.publisher || lookupInfo.publisher,
-      informative: spec.data.informative || trInfo.informative
+      informative: spec.data.informative || trInfo.informative,
+      milestones: spec.data.milestones || trInfo.milestones || {}
     };
 
     // Spec must have a title, either retrieved from Specref or defined in

tools/spec.jsons

@@ -180,6 +180,11 @@
           }
         }
       }
+    },
+    "milestones": {
+      "title": "Milestones to reach Recommendation",
+      "description": "Describes the current milestones (YYYY-MM-DD) as envisioned by the Working Group",
+      "type": "object"
     }
   }
 }

tools/tr.jsons

@@ -102,6 +102,11 @@
             }
           }
         }
+      },
+      "milestones": {
+        "title": "Milestones to reach Recommendation",
+        "description": "Describes the current milestones (YYYY-MM-DD) as envisioned by the Working Group",
+        "type": "object"
       }
     }
   }