Caching all markdown at build

.gitignore

@@ -9,6 +9,7 @@ package-lock.json
 # build output
 dist/
 .cache/
+src/content/posts/
 
 # generated types
 .astro/

__mocks__/astro:content.ts

@@ -0,0 +1,3 @@
+import { vi } from "vitest";
+
+export const getCollection = vi.fn();

astro.config.ts

@@ -1,6 +1,6 @@
 import { defineConfig } from "astro/config";
 import prefetch from "@astrojs/prefetch";
-import getRedirects from "./src/lib/getRedirects";
+// import getRedirects from "./src/lib/getRedirects";
 
 import sitemap from "@astrojs/sitemap";
 import "dotenv/config";
@@ -11,14 +11,17 @@ export default defineConfig({
   image: {
     domains: ["blog.beeminder.com", "user-images.githubusercontent.com"],
   },
-  redirects: await getRedirects().catch((e) => {
-    console.error(e);
-    throw e;
-  }),
+  // redirects: await getRedirects().catch((e) => {
+  //   console.error(e);
+  //   throw e;
+  // }),
   integrations: [
     prefetch({
       selector: "a",
     }),
     sitemap(),
   ],
+  experimental: {
+    contentCollectionCache: true,
+  },
 });

package.json

@@ -10,7 +10,7 @@
   "scripts": {
     "dev": "astro dev",
     "start": "astro dev",
-    "build": "tsx ./scripts/image-cache.ts && astro build",
+    "build": " pnpm md:pull && tsx ./scripts/image-cache.ts && astro build",
     "preview": "astro preview",
     "astro": "astro",
     "test": "vitest",
@@ -22,7 +22,8 @@
     "lint": "eslint .",
     "format": "prettier --write .",
     "preinstall": "npx only-allow pnpm",
-    "puppeteer": "tsx ./scripts/puppeteer.ts"
+    "puppeteer": "tsx ./scripts/puppeteer.ts",
+    "md:pull": "tsx ./scripts/pull.ts"
   },
   "dependencies": {
     "@astrojs/prefetch": "^0.4.1",
@@ -42,6 +43,7 @@
     "memize": "^2.1.0",
     "node-fetch-cache": "^3.1.3",
     "p-limit": "^4.0.0",
+    "sanitize-filename": "^1.6.3",
     "sanitize-html": "^2.11.0",
     "sharp": "^0.32.5",
     "striptags": "^3.2.0",

scripts/pull.ts

@@ -0,0 +1,26 @@
+import fs from "fs";
+import path from "path";
+import "dotenv/config";
+import fetchPosts from "../src/lib/fetchPosts";
+import sanitize from "sanitize-filename";
+
+const promises = fetchPosts();
+const dir = "./src/content/posts";
+
+console.log("Delete the directory if it does exist");
+if (fs.existsSync(dir)) {
+  fs.rmdirSync(dir, { recursive: true });
+}
+
+console.log("Create directory");
+fs.mkdirSync(dir, { recursive: true });
+
+promises.forEach((p) => {
+  p.then((post) => {
+    console.log(post.slug, post.source);
+    fs.writeFileSync(
+      path.join(dir, `${sanitize(String(post.source))}.json`),
+      JSON.stringify(post, null, 2),
+    );
+  });
+});

src/content/config.ts

@@ -0,0 +1,7 @@
+import { defineCollection } from "astro:content";
+
+export const collections = {
+  posts: defineCollection({
+    type: "data",
+  }),
+};

src/env.d.ts

@@ -1 +1,2 @@
+/// <reference path="../.astro/types.d.ts" />
 /// <reference types="astro/client" />

src/lib/env.ts

@@ -1,5 +1,5 @@
 // WORKAROUND: `import.meta.env` is not available during Astro config evaluation.
 // https://docs.astro.build/en/guides/configuring-astro/#environment-variables
 export default function env(key: string): string | undefined {
-  return import.meta.env[key] || process.env[key];
+  return import.meta.env?.[key] || process.env[key];
 }

src/lib/getArchives.spec.ts

@@ -2,10 +2,19 @@ import { describe, it, expect, beforeEach, vi } from "vitest";
 import getArchives from "./getArchives";
 import readSources from "./readSources";
 import meta from "./test/meta";
+import ether from "./test/ether";
+import { getCollection } from "astro:content";
 
 describe("getArchives", () => {
   beforeEach(() => {
-    vi.mocked(readSources).mockReturnValue([meta()]);
+    vi.mocked(getCollection).mockReturnValue([
+      {
+        data: {
+          ...meta(),
+          md: ether(),
+        },
+      },
+    ] as any);
   });
 
   it("batches by month", async () => {
@@ -32,18 +41,47 @@ describe("getArchives", () => {
   });
 
   it("handles three posts in one month", async () => {
-    vi.mocked(readSources).mockReturnValue([meta(), meta(), meta()]);
+    vi.mocked(getCollection).mockReturnValue([
+      {
+        data: {
+          ...meta(),
+          md: ether(),
+        },
+      },
+      {
+        data: {
+          ...meta(),
+          md: ether(),
+        },
+      },
+      {
+        data: {
+          ...meta(),
+          md: ether(),
+        },
+      },
+    ] as any);
 
     const result = await getArchives();
 
     expect(result[0]?.months[0]?.posts).toHaveLength(3);
   });
 
   it("sorts posts by date", async () => {
-    vi.mocked(readSources).mockReturnValue([
-      meta({ title: "A", date: "2013-02-22" }),
-      meta({ title: "B", date: "2013-02-21" }),
-    ]);
+    vi.mocked(getCollection).mockReturnValue([
+      {
+        data: {
+          ...meta({ title: "A", date: "2013-02-22" }),
+          md: ether(),
+        },
+      },
+      {
+        data: {
+          ...meta({ title: "B", date: "2013-02-21" }),
+          md: ether(),
+        },
+      },
+    ] as any);
 
     const result = await getArchives();