[METAED-1664] Choice optionality and dependency bumps

package.json

@@ -10,7 +10,7 @@
     "@typescript-eslint/parser": "8.58.2",
     "copyfiles": "^2.4.1",
     "cross-env": "^7.0.3",
-    "esbuild": "^0.28.0",
+    "esbuild": "^0.28.1",
     "eslint": "8.57.1",
     "eslint-config-airbnb-base": "15.0.0",
     "eslint-config-prettier": "8.5.0",
@@ -33,6 +33,20 @@
   "workspaces": [
     "packages/*"
   ],
+  "overrides": {
+    "@apidevtools/json-schema-ref-parser": {
+      "js-yaml": "4.3.0"
+    },
+    "cosmiconfig": {
+      "js-yaml": "4.3.0"
+    },
+    "ip-address": "10.2.0",
+    "lerna": {
+      "js-yaml": "4.3.0"
+    },
+    "sigstore": "4.1.1",
+    "tar": "7.5.19"
+  },
   "scripts": {
     "antlr-gen": "java -jar ./antlr/antlr-4.6-complete.jar -Dlanguage=JavaScript -o ./packages/metaed-core/src/grammar/gen ./packages/metaed-core/src/grammar/language/BaseLexer.g4 ./packages/metaed-core/src/grammar/language/MetaEdGrammar.g4",
     "metaed:build": "node ./packages/metaed-console/dist/index.js -c ./metaed.conf.json -a",

packages/metaed-core/package.json

@@ -20,7 +20,7 @@
     "deep-freeze": "^0.0.1",
     "deepmerge": "^4.3.1",
     "fs-extra": "^11.1.1",
-    "joi": "^17.10.1",
+    "joi": "^17.13.4",
     "klaw-sync": "^6.0.0",
     "ramda": "^0.32.0",
     "semver": "^7.3.8",

packages/metaed-plugin-edfi-api-catalog/package.json

@@ -19,7 +19,7 @@
     "write-excel-file": "^2.0.9"
   },
   "devDependencies": {
-    "read-excel-file": "^5.8.5"
+    "read-excel-file": "^9.2.0"
   },
   "scripts": {
     "build": "npm run build:clean && npm run build:dist",

packages/metaed-plugin-edfi-api-catalog/test/integration/ApiCatalogGenerator.test.ts

@@ -18,8 +18,8 @@ import {
 } from '@edfi/metaed-core';
 import { initialize as initializeUnifiedPlugin } from '@edfi/metaed-plugin-edfi-unified';
 import { initialize as initializeApiSchemaPlugin } from '@edfi/metaed-plugin-edfi-api-schema';
-import readXlsxFile from 'read-excel-file/node';
-import type { CellValue, Row } from 'read-excel-file';
+import { readSheet } from 'read-excel-file/node';
+import type { CellValue, Row } from 'read-excel-file/node';
 import { generate } from '../../src/generator/ApiCatalogGenerator';
 import { propertiesWorksheetName, resourcesWorksheetName } from '../../src/model/ApiCatalogRow';
 
@@ -124,15 +124,11 @@ describe('when generating api catalog workbook for real-model edge cases', (): v
     generatorResults = await generate(metaEd);
 
     resourceRows = toWorksheetRows(
-      await readXlsxFile(generatorResults.generatedOutput[0].resultStream ?? Buffer.alloc(0), {
-        sheet: resourcesWorksheetName,
-      }),
+      await readSheet(generatorResults.generatedOutput[0].resultStream ?? Buffer.alloc(0), resourcesWorksheetName),
     );
 
     propertyRows = toWorksheetRows(
-      await readXlsxFile(generatorResults.generatedOutput[0].resultStream ?? Buffer.alloc(0), {
-        sheet: propertiesWorksheetName,
-      }),
+      await readSheet(generatorResults.generatedOutput[0].resultStream ?? Buffer.alloc(0), propertiesWorksheetName),
     );
   });
 

packages/metaed-plugin-edfi-api-schema/package.json

@@ -19,7 +19,7 @@
     "ajv": "^8.12.0",
     "deep-freeze": "^0.0.1",
     "inflection": "^2.0.1",
-    "joi": "^18.0.1",
+    "joi": "^18.2.3",
     "json-stable-stringify": "^1.1.1",
     "ramda": "^0.32.0",
     "ts-invariant": "^0.10.3"

packages/metaed-plugin-edfi-api-schema/src/enhancer/AllJsonPathsMappingEnhancer.ts

@@ -72,6 +72,15 @@ function hasAtMostTwoArrayLevels(jsonPath: JsonPath): boolean {
   return matches.length <= 2;
 }
 
+/**
+ * Determines whether the value at this JsonPath is required within its containing JSON object or array item.
+ * Only flattened parent context carried in the PropertyModifier, such as choice or inline common optionality,
+ * changes that value. Required collection containers are handled by schema/OpenAPI requiredness, not here.
+ */
+function isJsonPathValueRequired(property: EntityProperty, propertyModifier: PropertyModifier): boolean {
+  return (property.isRequired || property.isPartOfIdentity) && !propertyModifier.optionalDueToParent;
+}
+
 /**
  * Adds a JsonPath to the JsonPathsMapping for a given list of PropertyPaths. Handles array initialization when needed.
  */
@@ -82,6 +91,7 @@ function addJsonPathTo(
   jsonPath: JsonPath,
   isTopLevel: boolean,
   terminalProperty: EntityProperty,
+  propertyModifier: PropertyModifier,
   flattenedIdentityProperty: FlattenedIdentityProperty,
   isArrayIdentity: boolean,
   collectionContainerJsonPath: JsonPath | null = null,
@@ -90,16 +100,21 @@ function addJsonPathTo(
   invariant(hasAtMostTwoArrayLevels(jsonPath), 'ApiSchema does not support documents wih arrays nested more than two deep');
 
   propertyPaths.forEach((propertyPath) => {
+    const isRequired: boolean = isJsonPathValueRequired(terminalProperty, propertyModifier);
+
     // initialize if necessary
     if (jsonPathsMapping[propertyPath] == null) {
       const initialJsonPathsInfo: JsonPathsInfo = isTopLevel
-        ? { jsonPathPropertyPairs: [], isTopLevel, isArrayIdentity, initialPropertyPath, terminalProperty }
-        : { jsonPathPropertyPairs: [], isTopLevel, isArrayIdentity, initialPropertyPath };
+        ? { jsonPathPropertyPairs: [], isTopLevel, isArrayIdentity, initialPropertyPath, terminalProperty, isRequired }
+        : { jsonPathPropertyPairs: [], isTopLevel, isArrayIdentity, initialPropertyPath, isRequired };
       jsonPathsMapping[propertyPath] = initialJsonPathsInfo;
     }
 
-    // Avoid duplicates
     const jsonPathsInfo: JsonPathsInfo = jsonPathsMapping[propertyPath];
+    invariant(
+      jsonPathsInfo.isRequired === isRequired,
+      `Conflicting isRequired detected for "${propertyPath}": "${jsonPathsInfo.isRequired}" vs "${isRequired}"`,
+    );
 
     if (collectionContainerJsonPath != null) {
       const existingContainer: JsonPath | undefined = jsonPathsInfo.collectionContainerJsonPath;
@@ -125,6 +140,7 @@ function addJsonPathTo(
       }
     }
 
+    // Avoid duplicates
     if (jsonPathsInfo.jsonPathPropertyPairs.map((jppp) => jppp.jsonPath).includes(jsonPath)) return;
 
     jsonPathsInfo.jsonPathPropertyPairs.push({
@@ -185,6 +201,7 @@ function jsonPathsForReferentialProperty(
     // Because these are flattened, we know they are non-reference properties
     jsonPathsForNonReference(
       flattenedIdentityProperty.identityProperty,
+      parentPropertyModifier(flattenedIdentityProperty, propertyModifier),
       jsonPathsMappingForThisProperty,
       initialPropertyPath,
       propertyPathsFromIdentityProperty(currentPropertyPath, flattenedIdentityProperty),
@@ -217,6 +234,7 @@ function jsonPathsForReferentialProperty(
               jsonPath,
               isTopLevel,
               property,
+              propertyModifier,
               flattenedIdentityProperty,
               isArrayIdentity,
               collectionContainerJsonPath,
@@ -406,6 +424,11 @@ function jsonPathsForChoiceAndInlineCommonProperty(
   collectionContainerJsonPath: JsonPath | null = null,
 ) {
   // prefixes from choice and inline common that affect child properties
+  const optionalDueToParent: boolean =
+    property.isOptional ||
+    property.isOptionalCollection ||
+    propertyModifier.optionalDueToParent ||
+    property.type === 'choice';
   const parentPrefixes: string[] =
     property.roleName === property.metaEdName
       ? [...propertyModifier.parentPrefixes]
@@ -415,7 +438,7 @@ function jsonPathsForChoiceAndInlineCommonProperty(
 
   allProperties.forEach((allProperty) => {
     const concatenatedPropertyModifier: PropertyModifier = propertyModifierConcat(
-      { optionalDueToParent: propertyModifier.optionalDueToParent, parentPrefixes },
+      { optionalDueToParent, parentPrefixes },
       allProperty.propertyModifier,
     );
 
@@ -445,6 +468,7 @@ function jsonPathsForChoiceAndInlineCommonProperty(
  */
 function jsonPathsForNonReference(
   property: EntityProperty,
+  propertyModifier: PropertyModifier,
   jsonPathsMapping: JsonPathsMapping,
   initialPropertyPath: MetaEdPropertyPath,
   currentPropertyPaths: MetaEdPropertyPath[],
@@ -465,6 +489,7 @@ function jsonPathsForNonReference(
       `${currentJsonPath}.schoolYear` as JsonPath,
       isTopLevel,
       property,
+      propertyModifier,
       flattenedIdentityProperty,
       isArrayIdentity,
       collectionContainerJsonPath,
@@ -477,6 +502,7 @@ function jsonPathsForNonReference(
       currentJsonPath,
       isTopLevel,
       property,
+      propertyModifier,
       flattenedIdentityProperty,
       isArrayIdentity,
       collectionContainerJsonPath,
@@ -539,6 +565,7 @@ function jsonPathsForDescriptorCollection(
     appendNextJsonPathName(collectionContainerJsonPath, apiMapping.descriptorCollectionName, property, propertyModifier),
     isTopLevel,
     property,
+    propertyModifier,
     NoFlattenedIdentityProperty,
     true,
     collectionContainerJsonPath,
@@ -564,6 +591,7 @@ function jsonPathsForNonReferenceCollection(
 
   jsonPathsForNonReference(
     property,
+    propertyModifier,
     jsonPathsMapping,
     initialPropertyPath,
     [currentPropertyPath],
@@ -585,6 +613,7 @@ function jsonPathsForNonReferenceCollection(
  */
 function jsonPathsForSchoolYearEnumeration(
   property: EntityProperty,
+  propertyModifier: PropertyModifier,
   jsonPathsMapping: JsonPathsMapping,
   initialPropertyPath: MetaEdPropertyPath,
   currentPropertyPath: MetaEdPropertyPath,
@@ -606,6 +635,7 @@ function jsonPathsForSchoolYearEnumeration(
     `${currentJsonPath}.schoolYear` as JsonPath,
     isTopLevel,
     property,
+    propertyModifier,
     NoFlattenedIdentityProperty,
     isArrayIdentity,
     collectionContainerJsonPath,
@@ -725,6 +755,7 @@ function jsonPathsFor(
   if (property.type === 'descriptor') {
     jsonPathsForNonReference(
       property,
+      propertyModifier,
       jsonPathsMapping,
       initialPropertyPath,
       [`${currentPropertyPath}Descriptor` as MetaEdPropertyPath],
@@ -739,6 +770,7 @@ function jsonPathsFor(
 
   jsonPathsForNonReference(
     property,
+    propertyModifier,
     jsonPathsMapping,
     initialPropertyPath,
     [currentPropertyPath],
@@ -780,6 +812,7 @@ function buildJsonPathsMapping(entity: TopLevelEntity) {
     if (property.type === 'schoolYearEnumeration')
       jsonPathsForSchoolYearEnumeration(
         property,
+        propertyModifier,
         allJsonPathsMapping,
         property.fullPropertyName as MetaEdPropertyPath,
         property.fullPropertyName as MetaEdPropertyPath,

packages/metaed-plugin-edfi-api-schema/src/enhancer/DocumentPathsMappingEnhancer.ts

@@ -224,6 +224,7 @@ function buildDocumentReferencePaths(
   startOfPropertyPath: string,
   referenceProperty: ReferentialProperty,
   allJsonPathsMapping: JsonPathsMapping,
+  isRequired: boolean,
 ): DocumentReferencePaths {
   const { referencedEntity } = referenceProperty;
   const referencedEntityApiSchemaData = referencedEntity.data.edfiApiSchema as EntityApiSchemaData;
@@ -239,14 +240,15 @@ function buildDocumentReferencePaths(
       allJsonPathsMapping,
     ),
     sourceProperty: referenceProperty,
-    isRequired: referenceProperty.isRequired || referenceProperty.isPartOfIdentity,
+    isRequired,
     isPartOfIdentity: referenceProperty.isPartOfIdentity,
   };
 }
 
 function buildDescriptorPath(
   jsonPathPropertyPairs: JsonPathPropertyPair[],
   property: EntityProperty,
+  isRequired: boolean,
 ): DescriptorReferencePath {
   invariant(jsonPathPropertyPairs.length === 1, 'Descriptor should only have one path');
 
@@ -261,13 +263,15 @@ function buildDescriptorPath(
     path: jsonPathPropertyPairs[0].jsonPath,
     type: getPathType(jsonPathPropertyPairs[0].sourceProperty.type),
     sourceProperty: jsonPathPropertyPairs[0].sourceProperty,
-    isRequired:
-      jsonPathPropertyPairs[0].sourceProperty.isRequired || jsonPathPropertyPairs[0].sourceProperty.isPartOfIdentity,
+    isRequired,
     isPartOfIdentity: jsonPathPropertyPairs[0].sourceProperty.isPartOfIdentity,
   };
 }
 
-function buildSchoolYearEnumerationPath(jsonPathPropertyPairs: JsonPathPropertyPair[]): DocumentReferencePaths {
+function buildSchoolYearEnumerationPath(
+  jsonPathPropertyPairs: JsonPathPropertyPair[],
+  isRequired: boolean,
+): DocumentReferencePaths {
   invariant(jsonPathPropertyPairs.length === 1, 'SchoolYear should only have one path');
 
   return {
@@ -283,21 +287,19 @@ function buildSchoolYearEnumerationPath(jsonPathPropertyPairs: JsonPathPropertyP
       },
     ],
     sourceProperty: jsonPathPropertyPairs[0].sourceProperty,
-    isRequired:
-      jsonPathPropertyPairs[0].sourceProperty.isRequired || jsonPathPropertyPairs[0].sourceProperty.isPartOfIdentity,
+    isRequired,
     isPartOfIdentity: jsonPathPropertyPairs[0].sourceProperty.isPartOfIdentity,
   };
 }
 
-function buildScalarPath(jsonPathPropertyPairs: JsonPathPropertyPair[]): ScalarPath {
+function buildScalarPath(jsonPathPropertyPairs: JsonPathPropertyPair[], isRequired: boolean): ScalarPath {
   invariant(jsonPathPropertyPairs.length === 1, 'Scalar should only have one path');
   return {
     path: jsonPathPropertyPairs[0].jsonPath,
     isReference: false,
     type: getPathType(jsonPathPropertyPairs[0].sourceProperty.type),
     sourceProperty: jsonPathPropertyPairs[0].sourceProperty,
-    isRequired:
-      jsonPathPropertyPairs[0].sourceProperty.isRequired || jsonPathPropertyPairs[0].sourceProperty.isPartOfIdentity,
+    isRequired,
     isPartOfIdentity: jsonPathPropertyPairs[0].sourceProperty.isPartOfIdentity,
   };
 }
@@ -323,13 +325,18 @@ function documentPathsMappingFor(entity: TopLevelEntity): DocumentPathsMapping {
 
     if (property.type === 'association' || property.type === 'domainEntity') {
       const referenceProperty: ReferentialProperty = property as ReferentialProperty;
-      result[propertyPath] = buildDocumentReferencePaths(propertyPath, referenceProperty, allJsonPathsMapping);
+      result[propertyPath] = buildDocumentReferencePaths(
+        propertyPath,
+        referenceProperty,
+        allJsonPathsMapping,
+        jsonPathsInfo.isRequired,
+      );
     } else if (property.type === 'descriptor') {
-      result[propertyPath] = buildDescriptorPath(jsonPathPropertyPairs, property);
+      result[propertyPath] = buildDescriptorPath(jsonPathPropertyPairs, property, jsonPathsInfo.isRequired);
     } else if (property.type === 'schoolYearEnumeration') {
-      result[propertyPath] = buildSchoolYearEnumerationPath(jsonPathPropertyPairs);
+      result[propertyPath] = buildSchoolYearEnumerationPath(jsonPathPropertyPairs, jsonPathsInfo.isRequired);
     } else {
-      result[propertyPath] = buildScalarPath(jsonPathPropertyPairs);
+      result[propertyPath] = buildScalarPath(jsonPathPropertyPairs, jsonPathsInfo.isRequired);
     }
   });