Add opt-in incremental builds (--incremental)

README.md

@@ -66,12 +66,19 @@ Options:
   -p, --preprocessed       output FSH produced by preprocessing steps
   -r, --require-latest     exit with error if this is not the latest version of SUSHI (default: false)
   -s, --snapshot           generate snapshot in Structure Definition output (default: false)
+  --incremental            reuse unchanged results from the previous build to rebuild faster (or set SUSHI_INCREMENTAL=1)
   -c, --config <config>    override elements in sushi-config.yaml (supported: 'version', 'status', 'releaselabel') (eg: --config status:draft)
   -h, --help               display help for command
 ```
 
 See the [SUSHI documentation](https://fshschool.org/docs/sushi/) for detailed information on using SUSHI.
 
+### Incremental Builds
+
+With `--incremental`, SUSHI keeps what it imported from each FSH file and reuses it in the next build when the file and everything it used from other files are unchanged. The output and messages are the same as a normal build's. This is useful for tools that run SUSHI on every edit: setting the `SUSHI_INCREMENTAL` environment variable to `1` turns it on for every build, for example when the IG Publisher runs SUSHI.
+
+The cache is kept in a `.sushi-cache` folder in the project (it contains its own `.gitignore`), or in the folder named by the `SUSHI_CACHE_DIR` environment variable. It can be deleted at any time. Incremental builds are not used with `--preprocessed`.
+
 ## IG Generation
 
 SUSHI supports publishing implementation guides via the new template-based IG Publisher. The template-based publisher is still being developed by the FHIR community. See the [Guidance for HL7 IG Creation](https://build.fhir.org/ig/FHIR/ig-guidance/) for more details.

src/app.ts

@@ -12,6 +12,7 @@ import { exportFHIR, Package } from './export';
 import { IGExporter, loadPredefinedResources } from './ig';
 import { createFHIRDefinitions } from './fhirdefs';
 import { Configuration } from './fshtypes';
+import { IncrementalBuild } from './incremental/IncrementalBuild';
 import {
   logger,
   stats,
@@ -74,6 +75,10 @@ async function app() {
       false
     )
     .option('-s, --snapshot', 'generate snapshot in Structure Definition output', false)
+    .option(
+      '--incremental',
+      'reuse unchanged results from the previous build to rebuild faster (or set SUSHI_INCREMENTAL=1)'
+    )
     .option(
       '-c, --config <config>',
       "override elements in sushi-config.yaml (supported: 'version', 'status', 'releaselabel') (eg: --config status:draft)",
@@ -190,6 +195,10 @@ async function runBuild(input: string, program: OptionValues, helpText: string)
   if (program.config) {
     Object.entries(program.config).forEach(([k, v]) => logger.info(`  --config ${k}:${v}`));
   }
+  const incrementalRequested = program.incremental || process.env.SUSHI_INCREMENTAL === '1';
+  if (incrementalRequested) {
+    logger.info('  --incremental');
+  }
   logger.info(`  ${path.resolve(input || '.')}`);
 
   // The version check hits the network, so only wait on it where the result is needed
@@ -240,6 +249,14 @@ async function runBuild(input: string, program: OptionValues, helpText: string)
 
   let tank: FSHTank;
   let config: Configuration;
+  // Incremental builds cannot reproduce the preprocessed FSH output, so they are not used with it
+  if (incrementalRequested && program.preprocessed) {
+    logger.info('Incremental build: not used, as it cannot produce preprocessed FSH.');
+  }
+  const incremental =
+    incrementalRequested && !program.preprocessed
+      ? new IncrementalBuild(path.resolve(originalInput), getLocalSushiVersion())
+      : null;
 
   try {
     let rawFSH: RawFSH[];
@@ -263,7 +280,7 @@ async function runBuild(input: string, program: OptionValues, helpText: string)
     }
     config = readConfig(originalInput);
     updateConfig(config, program);
-    tank = fillTank(rawFSH, config);
+    tank = incremental ? incremental.importFSH(rawFSH, config) : fillTank(rawFSH, config);
     tank.checkDuplicateNameEntities();
   } catch (e) {
     // If no errors have been logged yet, log this exception so the user knows why we're exiting
@@ -313,9 +330,19 @@ async function runBuild(input: string, program: OptionValues, helpText: string)
   // }
 
   logger.info('Converting FSH to FHIR resources...');
-  const outPackage = exportFHIR(tank, defs);
+  let outPackage: Package;
+  if (incremental) {
+    // the cache is reusable whether it was turned on by the option or the environment variable
+    const buildOptions = { ...program };
+    delete buildOptions.incremental;
+    outPackage = incremental.exportFHIR(tank, defs, buildOptions, config);
+    incremental.logSummary();
+  } else {
+    outPackage = exportFHIR(tank, defs);
+  }
   const { skippedResources } = writeFHIRResources(outDir, outPackage, defs, program.snapshot);
   writeFSHIndex(outDir, outPackage, input, skippedResources);
+  incremental?.finish();
 
   if (program.preprocessed) {
     logger.info('Writing preprocessed FSH...');

src/export/CodeSystemExporter.ts

@@ -20,6 +20,7 @@ import { MasterFisher, assembleFSHPath, resolveSoftIndexing } from '../utils';
 import { InstanceExporter, Package } from '.';
 import { CannotResolvePathError, MismatchedTypeError } from '../errors';
 import { isEqual } from 'lodash';
+import { incrementalHooks } from '../incremental/hooks';
 
 // The position of each concept within its list of concepts, by code. setConcepts never adds a second concept
 // with a code that is already present, and only ever appends concepts, so a position stays correct once it
@@ -393,6 +394,15 @@ export class CodeSystemExporter {
     if (this.pkg.codeSystems.some(cs => cs.name === fshDefinition.name)) {
       return;
     }
+    if (incrementalHooks.session != null) {
+      return incrementalHooks.session.exportEntity('cs', this, fshDefinition, () =>
+        this.exportNewCodeSystem(fshDefinition)
+      );
+    }
+    return this.exportNewCodeSystem(fshDefinition);
+  }
+
+  private exportNewCodeSystem(fshDefinition: FshCodeSystem): CodeSystem {
     const codeSystem = new CodeSystem();
     const codeSystemSD = codeSystem.getOwnStructureDefinition(this.fisher);
     this.setMetadata(codeSystem, fshDefinition);
@@ -402,13 +412,10 @@ export class CodeSystemExporter {
     );
     this.setCaretPathRules(codeSystem, fshDefinition, codeSystemSD, conceptIndex);
 
-    // check for another code system with the same id
-    // see https://www.hl7.org/fhir/resource.html#id
-    if (this.pkg.codeSystems.some(cs => codeSystem.id === cs.id)) {
-      logger.error(
-        `Multiple code systems with id ${codeSystem.id}. Each code system must have a unique id.`,
-        fshDefinition.sourceInfo
-      );
+    if (incrementalHooks.session != null) {
+      incrementalHooks.session.check(() => this.checkForDuplicateId(codeSystem, fshDefinition));
+    } else {
+      this.checkForDuplicateId(codeSystem, fshDefinition);
     }
 
     cleanResource(codeSystem, (prop: string) => ['_sliceName', '_primitive'].includes(prop));
@@ -423,6 +430,19 @@ export class CodeSystemExporter {
     return codeSystem;
   }
 
+  /**
+   * Checks for another code system with the same id.
+   * See https://www.hl7.org/fhir/resource.html#id
+   */
+  checkForDuplicateId(codeSystem: CodeSystem, fshDefinition: FshCodeSystem): void {
+    if (this.pkg.codeSystems.some(cs => codeSystem.id === cs.id)) {
+      logger.error(
+        `Multiple code systems with id ${codeSystem.id}. Each code system must have a unique id.`,
+        fshDefinition.sourceInfo
+      );
+    }
+  }
+
   export(): Package {
     const codeSystems = this.tank.getAllCodeSystems();
     for (const cs of codeSystems) {

src/export/FHIRExporter.ts

@@ -8,6 +8,7 @@ import {
   MappingExporter
 } from '.';
 import { MasterFisher } from '../utils';
+import { incrementalHooks } from '../incremental/hooks';
 /**
  * FHIRExporter handles the processing of FSH documents, storing the FSH types within them as FHIR types.
  * FHIRExporter takes the Profiles and Extensions within the FSHDocuments of a FSHTank and returns them
@@ -42,12 +43,24 @@ export class FHIRExporter {
     this.instanceExporter.applyInsertRules();
     this.mappingExporter.applyInsertRules();
 
+    incrementalHooks.session?.beginExport({
+      tank: this.tank,
+      pkg: this.pkg,
+      fisher: this.fisher,
+      structureDefinitionExporter: this.structureDefinitionExporter,
+      codeSystemExporter: this.codeSystemExporter,
+      valueSetExporter: this.valueSetExporter,
+      instanceExporter: this.instanceExporter
+    });
+
     this.structureDefinitionExporter.export();
     this.codeSystemExporter.export();
     this.valueSetExporter.export();
     this.instanceExporter.export();
+    incrementalHooks.session?.enterPostExport();
     this.structureDefinitionExporter.applyDeferredRules();
     this.mappingExporter.export();
+    incrementalHooks.session?.endExport();
 
     return this.pkg;
   }

src/export/InstanceExporter.ts

@@ -51,14 +51,17 @@ import {
 } from 'lodash';
 import { AssignmentRule, AssignmentValueType, PathRule } from '../fshtypes/rules';
 import chalk from 'chalk';
+import { incrementalHooks } from '../incremental/hooks';
 
 export class InstanceExporter implements Fishable {
   sdCache: Map<string, StructureDefinition> = new Map();
   constructor(
     private readonly tank: FSHTank,
     private readonly pkg: Package,
     private readonly fisher: MasterFisher
-  ) {}
+  ) {
+    incrementalHooks.session?.noteNewInstanceExporter(this);
+  }
 
   /**
    * Checks if there exist Instances of Custom Resources, which are not supported
@@ -809,7 +812,15 @@ export class InstanceExporter implements Fishable {
     if (this.pkg.instances.some(i => i._instanceMeta.name === fshDefinition.name)) {
       return;
     }
+    if (incrementalHooks.session != null) {
+      return incrementalHooks.session.exportEntity('inst', this, fshDefinition, () =>
+        this.exportNewInstance(fshDefinition)
+      );
+    }
+    return this.exportNewInstance(fshDefinition);
+  }
 
+  private exportNewInstance(fshDefinition: Instance): InstanceDefinition {
     let isResource = true;
     const json = this.fisher.fishForFHIR(
       fshDefinition.instanceOf,
@@ -857,9 +868,11 @@ export class InstanceExporter implements Fishable {
     let instanceOfStructureDefinition: StructureDefinition;
     if (this.sdCache.has(json.url)) {
       instanceOfStructureDefinition = this.sdCache.get(json.url);
+      incrementalHooks.session?.noteSdCache(this, json.url, false);
     } else {
       instanceOfStructureDefinition = StructureDefinition.fromJSON(json);
       this.sdCache.set(instanceOfStructureDefinition.url, instanceOfStructureDefinition);
+      incrementalHooks.session?.noteSdCache(this, instanceOfStructureDefinition.url, true);
     }
 
     let instanceDef = new InstanceDefinition();
@@ -1014,10 +1027,22 @@ export class InstanceExporter implements Fishable {
       }
     }
 
-    // check for another instance of the same type with the same id
-    // see https://www.hl7.org/fhir/resource.html#id
-    // instanceDef has already been added to the package, so it's fine if it matches itself
-    // inline instances are not written to their own files, so those can be skipped in this
+    if (incrementalHooks.session != null) {
+      incrementalHooks.session.check(() => this.checkForDuplicateId(instanceDef, fshDefinition));
+    } else {
+      this.checkForDuplicateId(instanceDef, fshDefinition);
+    }
+
+    return instanceDef;
+  }
+
+  /**
+   * Checks for another instance of the same type with the same id.
+   * See https://www.hl7.org/fhir/resource.html#id
+   * instanceDef has already been added to the package, so it's fine if it matches itself.
+   * Inline instances are not written to their own files, so those can be skipped in this check.
+   */
+  checkForDuplicateId(instanceDef: InstanceDefinition, fshDefinition: Instance): void {
     if (
       instanceDef._instanceMeta.usage !== 'Inline' &&
       this.pkg.instances.some(
@@ -1036,8 +1061,6 @@ export class InstanceExporter implements Fishable {
         fshDefinition.sourceInfo
       );
     }
-
-    return instanceDef;
   }
 
   /**