data-checks, upd8: report content string reference & key errors

2 files changed, 292 insertions, 7 deletions

diff --git a/src/data/checks.js b/src/data/checks.js
index 336cd64c..2c6ea99a 100644
--- a/src/data/checks.js
+++ b/src/data/checks.js
@@ -4,12 +4,15 @@ import {inspect as nodeInspect} from 'node:util';
 import {colors, ENABLE_COLOR} from '#cli';
 
 import CacheableObject from '#cacheable-object';
-import {compareArrays, empty, getNestedProp} from '#sugar';
+import {replacerSpec, parseInput} from '#replacer';
+import {compareArrays, cut, cutStart, empty, getNestedProp, iterateMultiline}
+  from '#sugar';
 import Thing from '#thing';
 import thingConstructors from '#things';
 import {commentaryRegexCaseSensitive} from '#wiki-data';
 
 import {
+  annotateErrorWithIndex,
   conditionallySuppressError,
   decorateErrorWithIndex,
   filterAggregate,
@@ -203,10 +206,8 @@ export function filterReferenceErrors(wikiData, {
   const aggregate = openAggregate({message: `Errors validating between-thing references in data`});
   for (const [thingDataProp, propSpec] of referenceSpec) {
     const thingData = getNestedProp(wikiData, thingDataProp);
-
+    const things = Array.isArray(thingData) ? thingData : [thingData];
     aggregate.nest({message: `Reference errors in ${colors.green('wikiData.' + thingDataProp)}`}, ({nest}) => {
-      const things = Array.isArray(thingData) ? thingData : [thingData];
-
       for (const thing of things) {
         nest({message: `Reference errors in ${inspect(thing)}`}, ({nest, push, filter}) => {
           for (const [property, findFnKey] of Object.entries(propSpec)) {
@@ -417,3 +418,252 @@ export function filterReferenceErrors(wikiData, {
 
   return aggregate;
 }
+
+export class ContentNodeError extends Error {
+  constructor({
+    length,
+    columnNumber,
+    containingLine,
+    where,
+    message,
+  }) {
+    const headingLine =
+      `(${where}) ${message}`;
+
+    const textUpToNode =
+      containingLine.slice(0, columnNumber);
+
+    const nodeText =
+      containingLine.slice(columnNumber, columnNumber + length);
+
+    const textPastNode =
+      containingLine.slice(columnNumber + length);
+
+    const containingLines =
+      containingLine.split('\n');
+
+    const formattedSourceLines =
+      containingLines.map((_, index, {length}) => {
+        let line = ' ⋮ ';
+
+        if (index === 0) {
+          line += colors.dim(cutStart(textUpToNode, 20));
+        }
+
+        line += nodeText;
+
+        if (index === length - 1) {
+          line += colors.dim(cut(textPastNode, 20));
+        }
+
+        return line;
+      });
+
+    super([
+      headingLine,
+      ...formattedSourceLines,
+    ].filter(Boolean).join('\n'));
+  }
+}
+
+export function reportContentTextErrors(wikiData, {
+  bindFind,
+}) {
+  const commentaryShape = {
+    body: 'commentary body',
+    artistDisplayText: 'commentary artist display text',
+    annotation: 'commentary annotation',
+  };
+
+  const contentTextSpec = [
+    ['albumData', {
+      commentary: commentaryShape,
+    }],
+
+    ['artistData', {
+      contextNotes: '_content',
+    }],
+
+    ['flashActData', {
+      jump: '_content',
+      listTerminology: '_content',
+    }],
+
+    ['groupData', {
+      description: '_content',
+    }],
+
+    ['homepageLayout', {
+      sidebarContent: '_content',
+    }],
+
+    ['newsData', {
+      content: '_content',
+    }],
+
+    ['staticPageData', {
+      content: '_content',
+    }],
+
+    ['trackData', {
+      commentary: commentaryShape,
+      lyrics: '_content',
+    }],
+
+    ['wikiInfo', {
+      description: '_content',
+      footerContent: '_content',
+    }],
+  ];
+
+  const boundFind = bindFind(wikiData, {mode: 'error'});
+  const findArtistOrAlias = bindFindArtistOrAlias(boundFind);
+
+  function* processContent(input) {
+    const nodes = parseInput(input);
+
+    for (const node of nodes) {
+      const index = node.i;
+      const length = node.iEnd - node.i;
+
+      if (node.type === 'tag') {
+        const replacerKeyImplied = !node.data.replacerKey;
+        const replacerKey = replacerKeyImplied ? 'track' : node.data.replacerKey.data;
+        const spec = replacerSpec[replacerKey];
+
+        if (!spec) {
+          yield {
+            index, length,
+            message:
+              `Unknown tag key ${colors.red(`"${replacerKey}"`)}`,
+          };
+        }
+
+        const replacerValue = node.data.replacerValue[0].data;
+
+        if (spec.find) {
+          let findFn;
+
+          switch (spec.find) {
+            case 'artist':
+              findFn = findArtistOrAlias;
+              break;
+
+            default:
+              findFn = boundFind[spec.find];
+              break;
+          }
+
+          const findRef =
+            (replacerKeyImplied
+              ? replacerValue
+              : replacerKey + `:` + replacerValue);
+
+          try {
+            findFn(findRef);
+          } catch (error) {
+            yield {
+              index, length,
+              message: error.message,
+            };
+          }
+        }
+      }
+    }
+  }
+
+  function callProcessContent({
+    nest,
+    push,
+    value,
+    message,
+    annotateError = error => error,
+  }) {
+    const processContentIterator =
+      nest({message}, ({call}) =>
+        call(processContent, value));
+
+    if (!processContentIterator) return;
+
+    const multilineIterator =
+      iterateMultiline(value, processContentIterator, {
+        formatWhere: true,
+        getContainingLine: true,
+      });
+
+    const errors = [];
+
+    for (const result of multilineIterator) {
+      errors.push(new ContentNodeError(result));
+    }
+
+    if (empty(errors)) return;
+
+    push(
+      annotateError(
+        new AggregateError(errors, message)));
+  }
+
+  withAggregate({message: `Errors validating content text`}, ({nest}) => {
+    for (const [thingDataProp, propSpec] of contentTextSpec) {
+      const thingData = getNestedProp(wikiData, thingDataProp);
+      const things = Array.isArray(thingData) ? thingData : [thingData];
+      nest({message: `Content text errors in ${colors.green('wikiData.' + thingDataProp)}`}, ({nest}) => {
+        for (const thing of things) {
+          nest({message: `Content text errors in ${inspect(thing)}`}, ({nest, push}) => {
+
+            for (const [property, shape] of Object.entries(propSpec)) {
+              const value = thing[property];
+
+              if (value === undefined) {
+                push(new TypeError(`Property ${colors.red(property)} isn't valid for ${colors.green(thing.constructor.name)}`));
+                continue;
+              }
+
+              if (value === null) {
+                continue;
+              }
+
+              const fieldPropertyMessage =
+                getFieldPropertyMessage(
+                  thing.constructor[Thing.yamlDocumentSpec],
+                  property);
+
+              const topMessage =
+                `Content text errors` + fieldPropertyMessage;
+
+              if (shape === '_content') {
+                callProcessContent({
+                  nest,
+                  push,
+                  value,
+                  message: topMessage,
+                });
+              } else {
+                nest({message: topMessage}, ({push}) => {
+                  for (const [index, entry] of value.entries()) {
+                    for (const [key, annotation] of Object.entries(shape)) {
+                      const value = entry[key];
+
+                      // TODO: Should this check undefined/null similar to above?
+                      if (!value) continue;
+
+                      callProcessContent({
+                        nest,
+                        push,
+                        value,
+                        message: `Error in ${colors.green(annotation)}`,
+                        annotateError: error =>
+                          annotateErrorWithIndex(error, index),
+                      });
+                    }
+                  }
+                });
+              }
+            }
+          });
+        }
+      });
+    }
+  });
+}
diff --git a/src/upd8.js b/src/upd8.js
index 24fba6ba..836ff6b2 100755
--- a/src/upd8.js
+++ b/src/upd8.js
@@ -42,8 +42,11 @@ import wrap from 'word-wrap';
 // order. This obviously needs fixing up.
 
 /* precede #find */
-import {filterReferenceErrors, reportDuplicateDirectories}
-  from '#data-checks';
+import {
+  filterReferenceErrors,
+  reportDuplicateDirectories,
+  reportContentTextErrors,
+} from '#data-checks';
 
 import {bindFind, getAllFindSpecs} from '#find';
 
@@ -138,11 +141,14 @@ async function main() {
       {...defaultStepStatus, name: `precache common data`},
 
     reportDuplicateDirectories:
-      {...defaultStepStatus, name: `filter duplicate directories`},
+      {...defaultStepStatus, name: `report duplicate directories`},
 
     filterReferenceErrors:
       {...defaultStepStatus, name: `filter reference errors`},
 
+    reportContentTextErrors:
+      {...defaultStepStatus, name: `report content text errors`},
+
     sortWikiDataArrays:
       {...defaultStepStatus, name: `sort wiki data arrays`},
 
@@ -1184,6 +1190,35 @@ async function main() {
     }
   }
 
+  if (stepStatusSummary.reportContentTextErrors.status === STATUS_NOT_STARTED) {
+    Object.assign(stepStatusSummary.reportContentTextErrors, {
+      status: STATUS_STARTED_NOT_DONE,
+      timeStart: Date.now(),
+    });
+
+    try {
+      reportContentTextErrors(wikiData, {bindFind});
+      logInfo`All content text validated without any errors - nice!`;
+
+      Object.assign(stepStatusSummary.reportContentTextErrors, {
+        status: STATUS_DONE_CLEAN,
+        timeEnd: Date.now(),
+      });
+    } catch (error) {
+      niceShowAggregate(error);
+
+      logWarn`The above errors were detected while processing content text in data files.`;
+      logWarn`The wiki will still build, but placeholders will be displayed in these spots.`;
+      logWarn`Resolve the errors for more complete output.`;
+
+      Object.assign(stepStatusSummary.reportContentTextErrors, {
+        status: STATUS_HAS_WARNINGS,
+        annotation: `view log for details`,
+        timeEnd: Date.now(),
+      });
+    }
+  }
+
   // Sort data arrays so that they're all in order! This may use properties
   // which are only available after the initial linking.