Merge branch 'main' into pre-expand-meta

.changeset/nine-kiwis-lie.md

@@ -0,0 +1,5 @@
+---
+'style-dictionary': patch
+---
+
+Fix bugs with expand tokens where they would run before instead of after user-configured preprocessors, and would fatally error on broken references. Broken refs should be tolerated at the expand stage, and errors will be thrown after preprocessor lifecycle if the refs are still broken at that point.

__tests__/StyleDictionary.test.js

@@ -347,6 +347,32 @@ describe('StyleDictionary class', () => {
   });
 
   describe('reference errors', () => {
+    // This is because some of those broken refs might get fixed in the preprocessor lifecycle hook by the user
+    // or by the built-in object-value token expand preprocessor
+    it('should tolerate broken references in the initialization phase', async () => {
+      let err;
+      let sd;
+      try {
+        sd = new StyleDictionary(
+          {
+            tokens: {
+              foo: {
+                value: '{bar}',
+                type: 'typography',
+              },
+            },
+            expand: true,
+          },
+          { init: false },
+        );
+
+        await sd.init();
+      } catch (e) {
+        err = e;
+      }
+      expect(err).to.be.undefined;
+    });
+
     it('should throw an error by default if broken references are encountered', async () => {
       const sd = new StyleDictionary({
         tokens: {
@@ -821,6 +847,56 @@ Use log.verbosity "verbose" or use CLI option --verbose for more details.
       await sd.hasInitialized;
       expect(sd.usesDtcg).to.be.true;
     });
+
+    it('should expand references when using DTCG format', async () => {
+      const sd = new StyleDictionary({
+        tokens: {
+          $type: 'typography',
+          typo: {
+            $value: {
+              fontSize: '16px',
+              fontWeight: 700,
+              fontFamily: 'Arial Black, sans-serif',
+            },
+          },
+          ref: {
+            $value: '{typo}',
+          },
+        },
+        expand: true,
+      });
+      await sd.hasInitialized;
+      expect(sd.tokens).to.eql({
+        typo: {
+          fontFamily: {
+            $type: 'fontFamily',
+            $value: 'Arial Black, sans-serif',
+          },
+          fontSize: {
+            $type: 'dimension',
+            $value: '16px',
+          },
+          fontWeight: {
+            $type: 'fontWeight',
+            $value: 700,
+          },
+        },
+        ref: {
+          fontFamily: {
+            $type: 'fontFamily',
+            $value: 'Arial Black, sans-serif',
+          },
+          fontSize: {
+            $type: 'dimension',
+            $value: '16px',
+          },
+          fontWeight: {
+            $type: 'fontWeight',
+            $value: 700,
+          },
+        },
+      });
+    });
   });
 
   describe('buildPlatform', () => {

docs/src/content/docs/info/architecture.md

@@ -67,6 +67,7 @@ Style Dictionary takes all the files it found and performs a deep merge. This al
 Allows users to configure [custom preprocessors](/reference/hooks/preprocessors), to process the merged dictionary as a whole, rather than per token file individually.
 These preprocessors have to be applied in the config, either on a global or platform level.
 Platform level preprocessors run once you get/export/format/build a platform, at the very start.
+Note that [tokens expansion](/reference/config#expand) runs after the user-configured preprocessors (for both global vs platform configured, respectively).
 
 ## 6. Transform the tokens
 

docs/src/content/docs/reference/config.md

@@ -262,11 +262,15 @@ The value of expand can be multiple things:
 
 You can enable the expanding of tokens both on a global level and on a platform level.
 
-One notable difference to keep in mind is that when you configure it on a global level, the token expansion will happen immediately **after** the [parsing hook](/reference/hooks/parsers) and **before** [preprocessing](/reference/hooks/preprocessors) or [transform](/reference/hooks/transforms) hooks.\
-This means that token metadata properties that are added by Style Dictionary such as `name`, `filePath`, `path`, `attributes` etc. are not present yet.\
-The advantage is having the expanded tokens (`sd.tokens` prop) available before doing any exporting to platforms.
+Whether configured on platform or global level, the token expansion will happen immediately **after** user-configured [preprocessors](/reference/hooks/preprocessors) and **before** [transform](/reference/hooks/transforms) hooks.\
+That said, platform expand happens only when calling `(get/export/format/build)Platform` methods for the specific platform, whereas global expand happens on StyleDictionary instantiation already.
 
-If you configure it on the platform level however, the metadata mentioned earlier is available and can be used to conditionally expand tokens.
+Refer to the [lifecycle hooks diagram](/info/architecture) for a better overview.
+
+When expanding globally, token metadata properties that are added by Style Dictionary such as `name`, `filePath`, `path`, `attributes` etc. are not present yet.\
+The advantage of global expand however, is having the expanded tokens (`sd.tokens` prop) available before doing any exporting to platforms.
+
+If you configure it on the platform level, the metadata mentioned earlier is available and can be used to conditionally expand tokens.
 It also allows you to expand tokens for some platforms but not for others.\
 The downside there is needing to configure it for every platform separately.
 

lib/StyleDictionary.js

@@ -331,25 +331,27 @@ export default class StyleDictionary extends Register {
         }
       }
     }
+    this.options = { ...this.options, usesDtcg: this.usesDtcg };
 
     // Merge inline, include, and source tokens
-    /** @type {PreprocessedTokens|Tokens} */
-    let tokens = deepExtend([{}, inlineTokens, includeTokens, sourceTokens]);
+    let preprocessedTokens = /** @type {PreprocessedTokens} */ (
+      deepExtend([{}, inlineTokens, includeTokens, sourceTokens])
+    );
+
+    preprocessedTokens = await preprocess(
+      preprocessedTokens,
+      this.preprocessors,
+      this.hooks.preprocessors,
+      this.options,
+    );
     if (this.usesDtcg) {
       // this is where they go from type Tokens -> Preprocessed tokens because the prop $type is removed
-      tokens = typeDtcgDelegate(tokens);
+      preprocessedTokens = typeDtcgDelegate(preprocessedTokens);
     }
-    let preprocessedTokens = /** @type {PreprocessedTokens} */ (tokens);
     if (this.shouldRunExpansion(this.expand)) {
       preprocessedTokens = expandTokens(preprocessedTokens, this.options);
     }
-    this.options = { ...this.options, usesDtcg: this.usesDtcg };
-    this.tokens = await preprocess(
-      preprocessedTokens,
-      this.preprocessors,
-      this.hooks.preprocessors,
-      this.options,
-    );
+    this.tokens = preprocessedTokens;
     this.hasInitializedResolve(null);
 
     // For chaining
@@ -391,14 +393,15 @@ export default class StyleDictionary extends Register {
     const platformConfig = transformConfig(this.platforms[platform], this, platform);
 
     let platformProcessedTokens = /** @type {PreprocessedTokens} */ (this.tokens);
-    if (this.shouldRunExpansion(platformConfig.expand)) {
-      platformProcessedTokens = expandTokens(platformProcessedTokens, this.options, platformConfig);
-    }
+
     platformProcessedTokens = await preprocess(
       platformProcessedTokens,
       platformConfig.preprocessors,
       this.hooks.preprocessors,
     );
+    if (this.shouldRunExpansion(platformConfig.expand)) {
+      platformProcessedTokens = expandTokens(platformProcessedTokens, this.options, platformConfig);
+    }
 
     let exportableResult = /** @type {PreprocessedTokens|TransformedTokens} */ (
       platformProcessedTokens

lib/utils/expandObjectTokens.js

@@ -222,7 +222,13 @@ function expandTokensRecurse(slice, original, opts, platform) {
     if (value) {
       // if our token is a ref, we have to resolve it first in order to expand its value
       if (typeof value === 'string' && usesReferences(value)) {
-        value = resolveReferences(value, original, { usesDtcg: uses$ });
+        try {
+          value = resolveReferences(value, original, { usesDtcg: uses$ });
+        } catch (e) {
+          // do nothing, references may be broken but now is not the time to
+          // complain about it, as we're just doing this here so we can expand
+          // tokens that reference object-value tokens that need to be expanded
+        }
       }
 
       if (