Attempt to reintroduce commonmark behavior

dev/lib/jsx-flow.js

@@ -2,6 +2,7 @@
  * @typedef {import('micromark-factory-mdx-expression').Acorn} Acorn
  * @typedef {import('micromark-factory-mdx-expression').AcornOptions} AcornOptions
  * @typedef {import('micromark-util-types').Construct} Construct
+ * @typedef {import('micromark-util-types').ConstructRecord} ConstructRecord
  * @typedef {import('micromark-util-types').State} State
  * @typedef {import('micromark-util-types').TokenizeContext} TokenizeContext
  * @typedef {import('micromark-util-types').Tokenizer} Tokenizer
@@ -14,6 +15,8 @@
  *   Acorn options.
  * @property {boolean | undefined} addResult
  *   Whether to add `estree` fields to tokens with results from acorn.
+ * @property {boolean} preferInline
+ *   Whether to parse text in flow elements as inline.
  */
 
 import {ok as assert} from 'devlop'
@@ -33,7 +36,9 @@
  *   Construct.
  */
 export function jsxFlow(acorn, options) {
-  return {name: 'mdxJsxFlowTag', tokenize: tokenizeJsxFlow, concrete: true}
+  const selfConstruct = {name: 'mdxJsxFlowTag', tokenize: tokenizeJsxFlow, concrete: !options.preferInline}
+
+  return selfConstruct
 
   /**
    * MDX JSX (flow).
@@ -49,6 +54,35 @@
   function tokenizeJsxFlow(effects, ok, nok) {
     const self = this
 
+    /**
+     * Search for a construct.
+     * @param {ConstructRecord[string]} where
+     * @param {string} name
+     * @returns {Construct | undefined}
+     */
+    function resolveConstruct(where, name) {
+      return [where].flat().find((d) => d?.name === name);
+    }
+
+    // We want to allow expressions directly after tags.
+    // See <https://github.com/micromark/micromark-extension-mdx-expression/blob/d5d92b9/packages/micromark-extension-mdx-expression/dev/lib/syntax.js#L183>
+    // for more info.
+    const leftBraceValue = self.parser.constructs.flow[codes.leftCurlyBrace]
+    /* c8 ignore next 5 -- always a list when normalized. */
+    const constructs = Array.isArray(leftBraceValue)
+      ? leftBraceValue
+      : leftBraceValue
+        ? [leftBraceValue]
+        : []
+    const mdxFlowExpression = resolveConstruct(
+      self.parser.constructs.flow[codes.leftCurlyBrace],
+      'mdxFlowExpression'
+    )
+    const mdxTextExpression = resolveConstruct(
+      self.parser.constructs.text[codes.leftCurlyBrace],
+      'mdxTextExpression'
+    )
+
     return start
 
     /**
@@ -143,27 +177,176 @@
      * @type {State}
      */
     function end(code) {
-      // We want to allow expressions directly after tags.
-      // See <https://github.com/micromark/micromark-extension-mdx-expression/blob/d5d92b9/packages/micromark-extension-mdx-expression/dev/lib/syntax.js#L183>
-      // for more info.
-      const leftBraceValue = self.parser.constructs.flow[codes.leftCurlyBrace]
-      /* c8 ignore next 5 -- always a list when normalized. */
-      const constructs = Array.isArray(leftBraceValue)
-        ? leftBraceValue
-        : leftBraceValue
-        ? [leftBraceValue]
-        : []
-      const expression = constructs.find((d) => d.name === 'mdxFlowExpression')
-
-      // Another tag.
-      return code === codes.lessThan
-        ? // We can’t just say: fine. Lines of blocks have to be parsed until an eol/eof.
-          start(code)
-        : code === codes.leftCurlyBrace && expression
-        ? effects.attempt(expression, end, nok)(code)
-        : code === codes.eof || markdownLineEnding(code)
-        ? ok(code)
-        : nok(code)
+      if (code === codes.eof) {
+        return ok(code);
+      }
+
+      if (!options.preferInline) {
+        if (code === codes.lessThan) {
+          // Another tag.
+          // We can’t just say: fine. Lines of blocks have to be parsed until an eol/eof.
+          return start(code);
+        } else if (code === codes.leftCurlyBrace && mdxFlowExpression) {
+          return effects.attempt(mdxFlowExpression, end, nok)(code);
+        } else if (markdownLineEnding(code)) {
+          return ok(code);
+        } else {
+          return nok(code);
+        }
+      } else {
+        if (markdownLineEnding(code)) {
+          effects.enter(types.lineEnding);
+          effects.consume(code);
+          effects.exit(types.lineEnding);
+          return newlineAfterTag;
+        } else {
+          return maybeText(code);
+        }
+      }
+    }
+
+    /**
+     * Handle content after newline following jsxFlowTag.
+     *
+     * ```markdown
+     *   | <A>
+     * > |
+     *     ^
+     * ```
+     *
+     * @type {State} 
+     */
+    function newlineAfterTag(code) {
+      if (markdownSpace(code)) {
+        // handle indent
+        return factorySpace(effects, newlineAfterTag, types.linePrefix)(code);
+      }
+
+      if (markdownLineEnding(code) || code === codes.eof) {
+        return ok(code);
+      }
+
+      return maybeText(code);
+    }
+
+    /**
+     * Handle something that might be text after jsxFlowTag.
+     *
+     * ```markdown
+     * > | <div>something
+     *          ^
+     * > | <div><span>
+     *          ^
+     * ```
+     *
+     * @type {State}
+     */
+    function maybeText(code) {
+      if (code === codes.eof) {
+        return ok(code);
+      }
+
+      if (code === codes.lessThan) {
+        // try next tag, or text
+        return effects.check(selfConstruct, start, textStart)(code);
+      }
+
+      if (code === codes.leftCurlyBrace && mdxTextExpression) {
+        return effects.attempt(mdxTextExpression, maybeText, textStart)(code);
+      }
+
+      if (markdownLineEnding(code)) {
+        // handle possible newline after expression
+        effects.enter('lineEnding');
+        effects.consume(code);
+        effects.exit('lineEnding');
+        return textNewlineContinuation;
+      }
+
+      return textStart(code);
+    }
+
+    /**
+     * Handle start of text.
+     *
+     * ```markdown
+     * > | <div>hello
+     *          ^
+     * ```
+     *
+     * @type {State}
+     */
+    function textStart(code) {
+      effects.enter('chunkText', { contentType: 'text' });
+      return textContinuation(code);
+    }
+
+    /**
+     * Handle text after jsxFlowTag.
+     *
+     * ```markdown
+     * > | <div>blah</div>
+     *          ^^^^
+     * ```
+     *
+     * @type {State}
+     */
+    function textContinuation(code) {
+      if (code === codes.lessThan) {
+        // try parse tag
+        return effects.check(
+          selfConstruct,
+          code => {
+            effects.exit('chunkText');
+            return start(code);
+          },
+          code => {
+            effects.consume(code);
+            return textContinuation;
+          }
+        )(code);
+      }
+
+      // </div>something (EOF) is an invalid flow block
+      if (code === codes.eof) {
+        return nok(code);
+      }
+
+      if (markdownLineEnding(code)) {
+        effects.exit('chunkText');
+        effects.enter('lineEnding');
+        effects.consume(code);
+        effects.exit('lineEnding');
+        return textNewlineContinuation;
+      }
+
+      // continue
+      effects.consume(code);
+      return textContinuation;
+    }
+
+    /**
+     * Handle after newline in textContinuation.
+     * 
+     * ```markdown
+     *   | <div>
+     *   |   hello
+     * > |
+     *     ^
+     * ```
+     *   
+     * @type {State}
+     */
+    function textNewlineContinuation(code) {
+      if (markdownSpace(code)) {
+        // handle indent
+        return factorySpace(effects, textNewlineContinuation, types.linePrefix)(code);
+      } else if (markdownLineEnding(code)) {
+        // cannot end block here
+        return nok(code);
+      } else {
+        return maybeText(code);
+      }
     }
   }
 }

dev/lib/syntax.js

@@ -15,6 +15,9 @@
  * @property {boolean | null | undefined} [addResult=false]
  *   Whether to add `estree` fields to tokens with results from acorn
  *   (default: `false`).
+ * @property {boolean | null | undefined} [preferInline=false]
+ *   If true, parse text in JSX flow tags as inline instead of block
+ *   (default: `false`). See CommonMark specification for more details.
  */
 
 import {codes} from 'micromark-util-symbol'
@@ -56,7 +59,8 @@ export function mdxJsx(options) {
     flow: {
       [codes.lessThan]: jsxFlow(acorn || undefined, {
         acornOptions,
-        addResult: settings.addResult || undefined
+        addResult: settings.addResult || undefined,
+        preferInline: settings.preferInline ?? false,
       })
     },
     text: {