path: root/src/data/checks.js

// checks.js - general validation and error/warning reporting for data objects

import {inspect as nodeInspect} from 'node:util';
import {colors, ENABLE_COLOR} from '#cli';

import CacheableObject from '#cacheable-object';
import {replacerSpec, parseInput} from '#replacer';
import {compareArrays, cut, cutStart, empty, getNestedProp, iterateMultiline}
  from '#sugar';
import Thing from '#thing';
import thingConstructors from '#things';
import {commentaryRegexCaseSensitive} from '#wiki-data';

import {
  annotateErrorWithIndex,
  conditionallySuppressError,
  decorateErrorWithIndex,
  filterAggregate,
  openAggregate,
  withAggregate,
} from '#aggregate';

function inspect(value, opts = {}) {
  return nodeInspect(value, {colors: ENABLE_COLOR, ...opts});
}

// Warn about directories which are reused across more than one of the same type
// of Thing. Directories are the unique identifier for most data objects across
// the wiki, so we have to make sure they aren't duplicated!
export function reportDuplicateDirectories(wikiData, {
  getAllFindSpecs,
}) {
  const duplicateSets = [];

  for (const findSpec of Object.values(getAllFindSpecs())) {
    if (!findSpec.bindTo) continue;

    const directoryPlaces = Object.create(null);
    const duplicateDirectories = new Set();
    const thingData = wikiData[findSpec.bindTo];

    for (const thing of thingData) {
      if (findSpec.include && !findSpec.include(thing)) {
        continue;
      }

      const directories =
        (findSpec.getMatchableDirectories
          ? findSpec.getMatchableDirectories(thing)
          : [thing.directory]);

      for (const directory of directories) {
        if (directory in directoryPlaces) {
          directoryPlaces[directory].push(thing);
          duplicateDirectories.add(directory);
        } else {
          directoryPlaces[directory] = [thing];
        }
      }
    }

    if (empty(duplicateDirectories)) continue;

    const sortedDuplicateDirectories =
      Array.from(duplicateDirectories)
        .sort((a, b) => {
          const aL = a.toLowerCase();
          const bL = b.toLowerCase();
          return aL < bL ? -1 : aL > bL ? 1 : 0;
        });

    for (const directory of sortedDuplicateDirectories) {
      const places = directoryPlaces[directory];
      duplicateSets.push({directory, places});
    }
  }

  if (empty(duplicateSets)) return;

  // Multiple find functions may effectively have duplicates across the same
  // things. These only need to be reported once, because resolving one of them
  // will resolve the rest, so cut out duplicate sets before reporting.

  const seenDuplicateSets = new Map();
  const deduplicateDuplicateSets = [];

  for (const set of duplicateSets) {
    if (seenDuplicateSets.has(set.directory)) {
      const placeLists = seenDuplicateSets.get(set.directory);

      for (const places of placeLists) {
        // We're iterating globally over all duplicate directories, which may
        // span multiple kinds of things, but that isn't going to cause an
        // issue because we're comparing the contents by identity, anyway.
        // Two artists named Foodog aren't going to match two tracks named
        // Foodog.
        if (compareArrays(places, set.places, {checkOrder: false})) {
          continue;
        }
      }

      placeLists.push(set.places);
    } else {
      seenDuplicateSets.set(set.directory, [set.places]);
    }

    deduplicateDuplicateSets.push(set);
  }

  withAggregate({message: `Duplicate directories found`}, ({push}) => {
    for (const {directory, places} of deduplicateDuplicateSets) {
      push(new Error(
        `Duplicate directory ${colors.green(`"${directory}"`)}:\n` +
        places.map(thing => ` - ` + inspect(thing)).join('\n')));
    }
  });
}

function bindFindArtistOrAlias(boundFind) {
  return artistRef => {
    const alias = boundFind.artistAlias(artistRef, {mode: 'quiet'});
    if (alias) {
      // No need to check if the original exists here. Aliases are automatically
      // created from a field on the original, so the original certainly exists.
      const original = alias.aliasedArtist;
      throw new Error(`Reference ${colors.red(artistRef)} is to an alias, should be ${colors.green(original.name)}`);
    }

    return boundFind.artist(artistRef);
  };
}

function getFieldPropertyMessage(yamlDocumentSpec, property) {
  const {fields} = yamlDocumentSpec;

  const field =
    Object.entries(fields ?? {})
      .find(([field, fieldSpec]) => fieldSpec.property === property)
      ?.[0];

  const fieldPropertyMessage =
    (field
      ? ` in field ${colors.green(field)}`
      : ` in property ${colors.green(property)}`);

  return fieldPropertyMessage;
}

// Warn about references across data which don't match anything.  This involves
// using the find() functions on all references, setting it to 'error' mode, and
// collecting everything in a structured logged (which gets logged if there are
// any errors). At the same time, we remove errored references from the thing's
// data array.
export function filterReferenceErrors(wikiData, {
  bindFind,
}) {
  const referenceSpec = [
    ['albumData', {
      artistContribs: '_contrib',
      coverArtistContribs: '_contrib',
      trackCoverArtistContribs: '_contrib',
      wallpaperArtistContribs: '_contrib',
      bannerArtistContribs: '_contrib',
      groups: 'group',
      artTags: '_artTag',
      commentary: '_commentary',
    }],

    ['groupCategoryData', {
      groups: 'group',
    }],

    ['homepageLayout.rows', {
      sourceGroup: '_homepageSourceGroup',
      sourceAlbums: 'album',
    }],

    ['flashData', {
      contributorContribs: '_contrib',
      featuredTracks: 'track',
    }],

    ['flashActData', {
      flashes: 'flash',
    }],

    ['trackData', {
      artistContribs: '_contrib',
      contributorContribs: '_contrib',
      coverArtistContribs: '_contrib',
      referencedTracks: '_trackNotRerelease',
      sampledTracks: '_trackNotRerelease',
      artTags: '_artTag',
      originalReleaseTrack: '_trackNotRerelease',
      commentary: '_commentary',
    }],

    ['wikiInfo', {
      divideTrackListsByGroups: 'group',
    }],
  ];

  const boundFind = bindFind(wikiData, {mode: 'error'});
  const findArtistOrAlias = bindFindArtistOrAlias(boundFind);

  const aggregate = openAggregate({message: `Errors validating between-thing references in data`});
  for (const [thingDataProp, propSpec] of referenceSpec) {
    const thingData = getNestedProp(wikiData, thingDataProp);
    const things = Array.isArray(thingData) ? thingData : [thingData];
    aggregate.nest({message: `Reference errors in ${colors.green('wikiData.' + thingDataProp)}`}, ({nest}) => {
      for (const thing of things) {
        nest({message: `Reference errors in ${inspect(thing)}`}, ({nest, push, filter}) => {
          for (const [property, findFnKey] of Object.entries(propSpec)) {
            let value = CacheableObject.getUpdateValue(thing, property);
            let writeProperty = true;

            switch (findFnKey) {
              case '_commentary':
                if (value) {
                  value =
                    Array.from(value.matchAll(commentaryRegexCaseSensitive))
                      .map(({groups}) => groups.artistReferences)
                      .map(text => text.split(',').map(text => text.trim()));
                }

                writeProperty = false;
                break;

              case '_contrib':
                // Don't write out contributions - these'll be filtered out
                // for content and data purposes automatically, and they're
                // handy to keep around when update values get checked for
                // art tags below. (Possibly no reference-related properties
                // need writing, humm...)
                writeProperty = false;
                break;
            }

            if (value === undefined) {
              push(new TypeError(`Property ${colors.red(property)} isn't valid for ${colors.green(thing.constructor.name)}`));
              continue;
            }

            if (value === null) {
              continue;
            }

            let findFn;

            switch (findFnKey) {
              case '_artTag':
                findFn = boundFind.artTag;
                break;

              case '_commentary':
                findFn = findArtistOrAlias;
                break;

              case '_contrib':
                findFn = contribRef => findArtistOrAlias(contribRef.who);
                break;

              case '_homepageSourceGroup':
                findFn = groupRef => {
                  if (groupRef === 'new-additions' || groupRef === 'new-releases') {
                    return true;
                  }

                  return boundFind.group(groupRef);
                };
                break;

              case '_trackNotRerelease':
                findFn = trackRef => {
                  const track = boundFind.track(trackRef);
                  const originalRef = track && CacheableObject.getUpdateValue(track, 'originalReleaseTrack');

                  if (originalRef) {
                    // It's possible for the original to not actually exist, in this case.
                    // It should still be reported since the 'Originally Released As' field
                    // was present.
                    const original = boundFind.track(originalRef, {mode: 'quiet'});

                    // Prefer references by name, but only if it's unambiguous.
                    const originalByName =
                      (original
                        ? boundFind.track(original.name, {mode: 'quiet'})
                        : null);

                    const shouldBeMessage =
                      (originalByName
                        ? colors.green(original.name)
                     : original
                        ? colors.green('track:' + original.directory)
                        : colors.green(originalRef));

                    throw new Error(`Reference ${colors.red(trackRef)} is to a rerelease, should be ${shouldBeMessage}`);
                  }

                  return track;
                };
                break;

              default:
                findFn = boundFind[findFnKey];
                break;
            }

            const suppress = fn => conditionallySuppressError(error => {
              if (property === 'sampledTracks') {
                // Suppress "didn't match anything" errors in particular, just for samples.
                // In hsmusic-data we have a lot of "stub" sample data which don't have
                // corresponding tracks yet, so it won't be useful to report such reference
                // errors until we take the time to address that. But other errors, like
                // malformed reference strings or miscapitalized existing tracks, should
                // still be reported, as samples of existing tracks *do* display on the
                // website!
                if (error.message.includes(`Didn't match anything`)) {
                  return true;
                }
              }

              return false;
            }, fn);

            const fieldPropertyMessage =
              getFieldPropertyMessage(
                thing.constructor[Thing.yamlDocumentSpec],
                property);

            const findFnMessage =
              (findFnKey.startsWith('_')
                ? ``
                : ` (${colors.green('find.' + findFnKey)})`);

            const errorMessage =
              (Array.isArray(value)
                ? `Reference errors` + fieldPropertyMessage + findFnMessage
                : `Reference error` + fieldPropertyMessage + findFnMessage);

            let newPropertyValue = value;

            determineNewPropertyValue: {
              // TODO: The special-casing for artTag is obviously a bit janky.
              // It would be nice if this could be moved to processDocument ala
              // fieldCombinationErrors, but art tags are only an error if the
              // thing doesn't have an artwork - which can't be determined from
              // the track document on its own, thanks to inheriting contribs
              // from the album.
              if (findFnKey === '_artTag') {
                let hasCoverArtwork =
                  !empty(CacheableObject.getUpdateValue(thing, 'coverArtistContribs'));

                if (thing.constructor === thingConstructors.Track) {
                  if (thing.album) {
                    hasCoverArtwork ||=
                      !empty(CacheableObject.getUpdateValue(thing.album, 'trackCoverArtistContribs'));
                  }

                  if (thing.disableUniqueCoverArt) {
                    hasCoverArtwork = false;
                  }
                }

                if (!hasCoverArtwork) {
                  nest({message: errorMessage}, ({push}) => {
                    push(new TypeError(`No cover artwork, so this shouldn't have art tags specified`));
                  });

                  newPropertyValue = [];
                  break determineNewPropertyValue;
                }
              }

              if (findFnKey === '_commentary') {
                filter(
                  value, {message: errorMessage},
                  decorateErrorWithIndex(refs =>
                    (refs.length === 1
                      ? suppress(findFn)(refs[0])
                      : filterAggregate(
                          refs, {message: `Errors in entry's artist references`},
                          decorateErrorWithIndex(suppress(findFn)))
                            .aggregate
                            .close())));

                // Commentary doesn't write a property value, so no need to set
                // anything on `newPropertyValue`.
                break determineNewPropertyValue;
              }

              if (Array.isArray(value)) {
                newPropertyValue = filter(
                  value, {message: errorMessage},
                  decorateErrorWithIndex(suppress(findFn)));
                break determineNewPropertyValue;
              }

              nest({message: errorMessage},
                suppress(({call}) => {
                  try {
                    call(findFn, value);
                  } catch (error) {
                    newPropertyValue = null;
                    throw error;
                  }
                }));
            }

            if (writeProperty) {
              thing[property] = newPropertyValue;
            }
          }
        });
      }
    });
  }

  return aggregate;
}

export class ContentNodeError extends Error {
  constructor({
    length,
    columnNumber,
    containingLine,
    where,
    message,
  }) {
    const headingLine =
      `(${where}) ${message}`;

    const textUpToNode =
      containingLine.slice(0, columnNumber);

    const nodeText =
      containingLine.slice(columnNumber, columnNumber + length);

    const textPastNode =
      containingLine.slice(columnNumber + length);

    const containingLines =
      containingLine.split('\n');

    const formattedSourceLines =
      containingLines.map((_, index, {length}) => {
        let line = ' ⋮ ';

        if (index === 0) {
          line += colors.dim(cutStart(textUpToNode, 20));
        }

        line += nodeText;

        if (index === length - 1) {
          line += colors.dim(cut(textPastNode, 20));
        }

        return line;
      });

    super([
      headingLine,
      ...formattedSourceLines,
    ].filter(Boolean).join('\n'));
  }
}

export function reportContentTextErrors(wikiData, {
  bindFind,
}) {
  const commentaryShape = {
    body: 'commentary body',
    artistDisplayText: 'commentary artist display text',
    annotation: 'commentary annotation',
  };

  const contentTextSpec = [
    ['albumData', {
      commentary: commentaryShape,
    }],

    ['artistData', {
      contextNotes: '_content',
    }],

    ['flashActData', {
      listTerminology: '_content',
    }],

    ['groupData', {
      description: '_content',
    }],

    ['homepageLayout', {
      sidebarContent: '_content',
    }],

    ['newsData', {
      content: '_content',
    }],

    ['staticPageData', {
      content: '_content',
    }],

    ['trackData', {
      commentary: commentaryShape,
      lyrics: '_content',
    }],

    ['wikiInfo', {
      description: '_content',
      footerContent: '_content',
    }],
  ];

  const boundFind = bindFind(wikiData, {mode: 'error'});
  const findArtistOrAlias = bindFindArtistOrAlias(boundFind);

  function* processContent(input) {
    const nodes = parseInput(input);

    for (const node of nodes) {
      const index = node.i;
      const length = node.iEnd - node.i;

      if (node.type === 'tag') {
        const replacerKeyImplied = !node.data.replacerKey;
        const replacerKey = replacerKeyImplied ? 'track' : node.data.replacerKey.data;
        const spec = replacerSpec[replacerKey];

        if (!spec) {
          yield {
            index, length,
            message:
              `Unknown tag key ${colors.red(`"${replacerKey}"`)}`,
          };

          // No spec, no further errors to report.
          continue;
        }

        const replacerValue = node.data.replacerValue[0].data;

        if (spec.find) {
          let findFn;

          switch (spec.find) {
            case 'artist':
              findFn = findArtistOrAlias;
              break;

            default:
              findFn = boundFind[spec.find];
              break;
          }

          const findRef =
            (replacerKeyImplied
              ? replacerValue
              : replacerKey + `:` + replacerValue);

          try {
            findFn(findRef);
          } catch (error) {
            yield {
              index, length,
              message: error.message,
            };

            // It's only possible to have one error per node at the moment.
            continue;
          }
        }
      } else if (node.type === 'external-link') {
        try {
          new URL(node.data.href);
        } catch (error) {
          yield {
            index, length,
            message:
              `Invalid URL ${colors.red(`"${node.data.href}"`)}`,
          };
        }
      }
    }
  }

  function callProcessContent({
    nest,
    push,
    value,
    message,
    annotateError = error => error,
  }) {
    const processContentIterator =
      nest({message}, ({call}) =>
        call(processContent, value));

    if (!processContentIterator) return;

    const multilineIterator =
      iterateMultiline(value, processContentIterator, {
        formatWhere: true,
        getContainingLine: true,
      });

    const errors = [];

    for (const result of multilineIterator) {
      errors.push(new ContentNodeError(result));
    }

    if (empty(errors)) return;

    push(
      annotateError(
        new AggregateError(errors, message)));
  }

  withAggregate({message: `Errors validating content text`}, ({nest}) => {
    for (const [thingDataProp, propSpec] of contentTextSpec) {
      const thingData = getNestedProp(wikiData, thingDataProp);
      const things = Array.isArray(thingData) ? thingData : [thingData];
      nest({message: `Content text errors in ${colors.green('wikiData.' + thingDataProp)}`}, ({nest}) => {
        for (const thing of things) {
          nest({message: `Content text errors in ${inspect(thing)}`}, ({nest, push}) => {

            for (const [property, shape] of Object.entries(propSpec)) {
              const value = thing[property];

              if (value === undefined) {
                push(new TypeError(`Property ${colors.red(property)} isn't valid for ${colors.green(thing.constructor.name)}`));
                continue;
              }

              if (value === null) {
                continue;
              }

              const fieldPropertyMessage =
                getFieldPropertyMessage(
                  thing.constructor[Thing.yamlDocumentSpec],
                  property);

              const topMessage =
                `Content text errors` + fieldPropertyMessage;

              if (shape === '_content') {
                callProcessContent({
                  nest,
                  push,
                  value,
                  message: topMessage,
                });
              } else {
                nest({message: topMessage}, ({push}) => {
                  for (const [index, entry] of value.entries()) {
                    for (const [key, annotation] of Object.entries(shape)) {
                      const value = entry[key];

                      // TODO: Should this check undefined/null similar to above?
                      if (!value) continue;

                      callProcessContent({
                        nest,
                        push,
                        value,
                        message: `Error in ${colors.green(annotation)}`,
                        annotateError: error =>
                          annotateErrorWithIndex(error, index),
                      });
                    }
                  }
                });
              }
            }
          });
        }
      });
    }
  });
}