Exploring Mutate

packages/core/src/generators/schema.ts

@@ -7,11 +7,14 @@ import { VFile } from 'vfile';
 import { cacheSchema, checkCacheForSchema } from '../cache/cache';
 import {
   CollectionContext,
+  CollectionEntry,
   ConfigResult,
   EntryNode,
   LoadedFlatbreadConfig,
+  Source,
   Transformer,
 } from '../types';
+import createHash from '../utils/createHash';
 import { getFieldOverrides } from '../utils/fieldOverrides';
 import { map } from '../utils/map';
 import addCollectionMutations from './collectionMutations';
@@ -41,7 +44,27 @@ export async function generateSchema(
   config.source.initialize?.(config);
 
   // Invoke the content source resolver to retrieve the content nodes
-  const allContentNodes = await config.source.fetch(config.content);
+  let allContentNodes: Record<string, any> = {};
+
+  const addRecord =
+    (source: Source<any>) =>
+    <Ctx>(collection: CollectionEntry, record: EntryNode, context: Ctx) => {
+      allContentNodes[collection.collection] =
+        allContentNodes[collection.collection] ?? [];
+      allContentNodes[collection.collection].push({
+        record,
+        context: {
+          sourceContext: context,
+          sourcedBy: source.id,
+          collection: collection.collection,
+          referenceField: collection.referenceField ?? 'id',
+        },
+      });
+    };
+
+  await config.source.fetch(config.content, {
+    addRecord: addRecord(config.source),
+  });
 
   // Transform the content nodes to the expected JSON format if needed
   const allContentNodesJSON = optionallyTransformContentNodes(
@@ -81,15 +104,15 @@ export async function generateSchema(
   };
 
   async function updateCollectionRecord(
-    entry: EntryNode & { _flatbread: CollectionContext }
+    entry: EntryNode & { _flatbread: any }
   ) {
     const { _flatbread: ctx, ...record } = entry;
     const file = await transformersById[ctx.transformedBy].serialize(
       record,
-      ctx
+      ctx.transformContext
     );
 
-    await config?.source.put(file, ctx);
+    await config?.source.put(file, ctx.sourceContext);
     const index = allContentNodesJSON[ctx.collection].findIndex(
       (c) => get(c, ctx.referenceField) === ctx.reference
     );
@@ -232,14 +255,16 @@ const optionallyTransformContentNodes = (
      * @todo if this becomes a performance bottleneck, consider overloading the source plugin API to accept a transform function so we can avoid mapping through the content nodes twice
      * */
 
-    return map(allContentNodes, (node: VFile) => {
-      const transformer = transformerMap.get(node.extname);
+    return map(allContentNodes, (node: { record: VFile; context: any }) => {
+      const transformer = transformerMap.get(node.record.extname);
       if (!transformer?.parse) {
-        throw new Error(`no transformer found for ${node.path}`);
+        throw new Error(`no transformer found for ${node.record.path}`);
       }
-      const doc = transformer.parse(node);
+      const { record: doc, context } = transformer.parse(node.record);
+      doc._flatbread = node.context;
       doc._flatbread.transformedBy = transformer.id;
-      doc._flatbread.reference = get(doc, doc._flatbread.referenceField);
+      doc._flatbread.transformContext = context;
+      doc._flatbread.reference = get(doc, node.context.referenceField);
       return doc;
     });
   }

packages/core/src/types.ts

@@ -21,13 +21,13 @@ export type ContentNode = BaseContentNode & {
  * @todo This needs to be typed more strictly.
  */
 export interface FlatbreadConfig {
-  source: Source;
+  source: Source<any>;
   transformer?: Transformer | Transformer[];
   content: Partial<CollectionEntry>[];
 }
 
 export interface LoadedFlatbreadConfig {
-  source: Source;
+  source: Source<any>;
   transformer: Transformer[];
   content: CollectionEntry[];
   loaded: {
@@ -69,16 +69,28 @@ export type EntryNode = Record<string, any>;
  * The result of an invoked `Source` plugin which contains methods on how to retrieve content nodes in
  * their raw (if coupled with a `Transformer` plugin) or processed form.
  */
-export interface Source {
+
+export interface FlatbreadArgs<Context> {
+  addRecord(
+    collection: CollectionEntry,
+    record: EntryNode,
+    context: Context
+  ): void;
+}
+
+export interface Source<Context> {
   initialize?: (flatbreadConfig: LoadedFlatbreadConfig) => void;
   id?: string;
-  put: (source: VFile, ctx: CollectionContext) => Promise<void>;
+  put: (source: VFile, ctx: Context) => Promise<Context>;
   fetch: (
-    allContentTypes: CollectionEntry[]
-  ) => Promise<Record<string, VFile[]>>;
+    allContentTypes: CollectionEntry[],
+    flatbread: FlatbreadArgs<Context>
+  ) => Promise<void>;
 }
 
-export type SourcePlugin = (sourceConfig?: Record<string, any>) => Source;
+export type SourcePlugin<Context> = (
+  sourceConfig?: Record<string, any>
+) => Source<Context>;
 
 /**
  * An override can be used to declare a custom resolve for a field in content

packages/core/src/utils/createHash.ts

@@ -0,0 +1,6 @@
+import { anyToString } from './stringUtils';
+import { createHash as createHashRaw } from 'crypto';
+
+export default function createHash(content: any) {
+  return createHashRaw('sha256').update(anyToString(content)).digest('hex');
+}

packages/core/src/utils/initializeConfig.ts

@@ -2,7 +2,7 @@ import { cloneDeep, defaultsDeep } from 'lodash-es';
 import { CollectionEntry } from '../types';
 import { FlatbreadConfig, LoadedFlatbreadConfig, Transformer } from '../types';
 import { toArray } from './arrayUtils';
-import { createHash } from 'crypto';
+import createHash from './createHash';
 import { anyToString } from './stringUtils';
 
 /**
@@ -13,10 +13,12 @@ export function initializeConfig(
 ): LoadedFlatbreadConfig {
   const config = cloneDeep(rawConfig);
   const transformer = toArray(config.transformer ?? []).map((t) => {
-    t.id = t.id ?? createHash('sha256').update(anyToString(t)).digest('hex');
+    t.id = t.id ?? createHash(t);
     return t;
   });
 
+  config.source.id = config.source.id ?? createHash(config.source);
+
   return {
     ...config,
     content: config.content?.map((content: Partial<CollectionEntry>) =>

packages/flatbread/content/authors/me.md

@@ -8,7 +8,7 @@ enjoys:
   - making this
 date_joined: 2021-02-25T16:41:59.558Z
 skills:
-  sitting: 23
+  sitting: 69
   breathing: 7.07
   liquid_consumption: 100
   existence: simulation

packages/source-filesystem/src/index.ts

@@ -5,6 +5,7 @@ import ownPackage from '../package.json' assert { type: 'json' };
 import type {
   CollectionContext,
   CollectionEntry,
+  FlatbreadArgs,
   LoadedFlatbreadConfig,
   SourcePlugin,
 } from '@flatbread/core';
@@ -17,6 +18,12 @@ import type {
 } from './types';
 import gatherFileNodes from './utils/gatherFileNodes';
 
+interface Context {
+  filename?: string;
+  path: string;
+  slug: string;
+}
+
 /**
  * Get nodes (files) from the directory
  *
@@ -26,28 +33,23 @@ import gatherFileNodes from './utils/gatherFileNodes';
  */
 async function getNodesFromDirectory(
   collectionEntry: CollectionEntry,
+  { addRecord }: FlatbreadArgs<Context>,
   config: InitializedSourceFilesystemConfig
-): Promise<VFile[]> {
+): Promise<void> {
   const { extensions } = config;
   const nodes: FileNode[] = await gatherFileNodes(collectionEntry.path, {
     extensions,
   });
 
-  return Promise.all(
-    nodes.map(async (node: FileNode): Promise<VFile> => {
-      const file = await read(node.path);
-      file.data = merge(node.data, {
-        _flatbread: {
-          referenceField: collectionEntry.referenceField,
-          collection: collectionEntry.collection,
-          filename: file.basename,
-          path: relative(process.cwd(), file.path),
-          slug: slugify(file.stem ?? ''),
-          sourcedBy: ownPackage.name,
-        },
+  await Promise.all(
+    nodes.map(async (node: FileNode): Promise<void> => {
+      const doc = await read(node.path);
+      doc.data = node.data;
+      addRecord(collectionEntry, doc, {
+        filename: doc.basename,
+        path: relative(process.cwd(), doc.path),
+        slug: slugify(doc.stem ?? ''),
       });
-
-      return file;
     })
   );
 }
@@ -60,15 +62,16 @@ async function getNodesFromDirectory(
  */
 async function getAllNodes(
   allCollectionEntries: CollectionEntry[],
+  flatbread: FlatbreadArgs<Context>,
   config: InitializedSourceFilesystemConfig
-): Promise<Record<string, VFile[]>> {
+): Promise<void> {
   const nodeEntries = await Promise.all(
     allCollectionEntries.map(
       async (contentType): Promise<Record<string, any>> =>
         new Promise(async (res) =>
           res([
             contentType.collection,
-            await getNodesFromDirectory(contentType, config),
+            await getNodesFromDirectory(contentType, flatbread, config),
           ])
         )
     )
@@ -77,19 +80,19 @@ async function getAllNodes(
   const nodes = Object.fromEntries(
     nodeEntries as Iterable<readonly [PropertyKey, any]>
   );
-
-  return nodes;
 }
 
 // TODO: _flatbread data should be extracted from plugins
 // plugin should return a context object and be given the same context object back when saving,
 // this context object will be saved internally under _flatbread[collectionId]
 
-async function put(source: VFile, ctx: CollectionContext) {
-  (source.basename = ctx.filename),
-    (source.path = resolve(process.cwd(), ctx.path));
+async function put(doc: VFile, context: Context) {
+  doc.basename = context.filename;
+  doc.path = resolve(process.cwd(), context.path);
 
-  await write(source);
+  await write(doc);
+
+  return { doc, context };
 }
 
 /**
@@ -98,17 +101,19 @@ async function put(source: VFile, ctx: CollectionContext) {
  * @param sourceConfig content types config
  * @returns A function that returns functions which fetch lists of nodes
  */
-export const source: SourcePlugin = (sourceConfig?: sourceFilesystemConfig) => {
+
+export function source(sourceConfig?: sourceFilesystemConfig) {
   let config: InitializedSourceFilesystemConfig;
 
   return {
     initialize: (flatbreadConfig: LoadedFlatbreadConfig) => {
       const { extensions } = flatbreadConfig.loaded;
       config = defaultsDeep(sourceConfig ?? {}, { extensions });
     },
-    fetch: (content: CollectionEntry[]) => getAllNodes(content, config),
+    fetch: (content: CollectionEntry[], flatbread: FlatbreadArgs<Context>) =>
+      getAllNodes(content, flatbread, config),
     put,
   };
-};
+}
 
 export default source;

packages/transformer-markdown/src/index.ts

@@ -24,10 +24,12 @@ export const parse = (
 ): EntryNode => {
   const { data, content } = matter(String(input), config.grayMatter);
   return {
-    ...input.data,
-    ...data,
-    _content: {
-      raw: content,
+    record: {
+      ...input.data,
+      ...data,
+      _content: {
+        raw: content,
+      },
     },
   };
 };

packages/transformer-yaml/src/index.ts

@@ -22,8 +22,10 @@ export const parse = (input: VFile): EntryNode => {
 
   if (typeof doc === 'object') {
     return {
-      ...input.data,
-      ...doc,
+      record: {
+        ...input.data,
+        ...doc,
+      },
     };
   }
   throw new Error(

packages/transformer-yaml/src/tests/index.test.ts

@@ -26,6 +26,6 @@ const transformer = Transformer();
 
 test('it can parse a basic yaml file', async (t) => {
   const parse = transformer.parse as (input: VFile) => EntryNode;
-  const node = parse(testFile);
-  t.snapshot(node);
+  const { record } = parse(testFile);
+  t.snapshot(record);
 });