[WIP] - Cache

defi/src/api2/cache/api-fetch-cache.ts

@@ -0,0 +1,146 @@
+import dotenv from 'dotenv';
+import fs from 'fs/promises';
+import * as HyperExpress from 'hyper-express';
+import fetch from 'node-fetch';
+import path from 'path';
+import { readFileData, storeData } from './file-cache';
+
+dotenv.config();
+
+const CACHE_SUBDIR = 'endpoints';
+const DEFAULT_TIMEOUT_MS = 30_000;
+const DEFAULT_TTL_MS = 60_000 * 60 * 6;
+const MIN_REFRESH_SEC_HARD = 5 * 60;
+const FAST_FALLBACK_MS = 10_000;
+
+const ALLOWED_HOSTS = new Set(['api.llama.fi', 'localhost:5001']);
+
+const CACHE_ROOT = process.env.API2_CACHE_DIR ?? path.join(process.cwd(), 'defi', 'src', 'api2', '.api2-cache');
+const ENDPOINT_DIR = path.join(CACHE_ROOT, CACHE_SUBDIR);
+
+const flatten = (s: string) => s.replace(/[^a-zA-Z0-9]/g, '_');
+
+const baseKey = (raw: string) => {
+  const { pathname } = new URL(raw);
+  return flatten(pathname.replace(/^\/+/u, ''));
+};
+
+const isErrorPayload = (d: unknown): d is { error: unknown } => typeof d === 'object' && d !== null && 'error' in d;
+
+const isAllowedHost = (raw: string) => {
+  try {
+    return ALLOWED_HOSTS.has(new URL(raw).host);
+  } catch {
+    return false;
+  }
+};
+
+const ensureDir = async () => fs.mkdir(ENDPOINT_DIR, { recursive: true });
+
+const purgeOld = async (base: string) => {
+  await ensureDir();
+  const files = await fs.readdir(ENDPOINT_DIR);
+  await Promise.all(
+    files
+      .filter((f) => f.startsWith(base + '-'))
+      .map((f) => fs.unlink(path.join(ENDPOINT_DIR, f)).catch(() => {})),
+  );
+};
+
+const latestFile = async (base: string) => {
+  await ensureDir();
+  const files = await fs.readdir(ENDPOINT_DIR);
+  return (
+    files
+      .filter((f) => f.startsWith(base + '-'))
+      .map((f) => {
+        const ts = Number(f.slice(base.length + 1, -5));
+        return { file: f, ts };
+      })
+      .filter((x) => !isNaN(x.ts))
+      .sort((a, b) => b.ts - a.ts)[0] ?? null
+  );
+};
+
+async function fetchWithTimeout(url: string, ms: number) {
+  const ctrl = new AbortController();
+  const timer = setTimeout(() => ctrl.abort(), ms);
+  try {
+    const r = await fetch(url, { signal: ctrl.signal });
+    if (!r.ok) throw new Error(`HTTP ${r.status}`);
+    const ct = r.headers.get('content-type') ?? '';
+    return ct.includes('application/json') ? await r.json() : await r.text();
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+export default function setApiFetchCacheRoute(router: HyperExpress.Router) {
+  router.get('/cache', async (req, res) => {
+    const api = req.query_parameters.api;
+    if (!api) return res.status(400).json({ error: 'api query param missing' });
+
+    if (!isAllowedHost(api))
+      return res.status(400).json({ error: 'host not allowed' });
+
+    const timeoutMs = Number(req.query_parameters.timeout ?? DEFAULT_TIMEOUT_MS / 1_000) * 1_000;
+
+    const ttlMs = DEFAULT_TTL_MS;
+    const minRefreshMs = MIN_REFRESH_SEC_HARD * 1_000;
+
+    const base = baseKey(api);
+    const latest = await latestFile(base);
+
+    const liveFetch = fetchWithTimeout(api, timeoutMs);
+
+    let responded = false;
+
+    const fallbackTimer = setTimeout(async () => {
+      if (responded) return;
+
+      if (latest && Date.now() - latest.ts * 1_000 < ttlMs) {
+        responded = true;
+        res.setHeader('X-Cache', 'HIT-STALE');
+        const cached = await readFileData(path.join(CACHE_SUBDIR, latest.file));
+        res.json(cached.data);
+      }
+    }, FAST_FALLBACK_MS);
+
+    try {
+      const data = await liveFetch;
+      clearTimeout(fallbackTimer);
+
+      if (!responded) {
+        responded = true;
+        res.setHeader('X-Cache', 'MISS');
+        res.json(data);
+      }
+
+      const isEmptyObj = typeof data === 'object' && data !== null && Object.keys(data as object).length === 0;
+      const isEmpty = data === null || data === undefined || isEmptyObj;
+      const ageOk = !latest || Date.now() - latest.ts * 1_000 > minRefreshMs;
+
+      if (!isEmpty && !isErrorPayload(data) && ageOk) {
+        await purgeOld(base);
+        const tsSec = Math.floor(Date.now() / 1_000);
+        await storeData(path.join(CACHE_SUBDIR, `${base}-${tsSec}.json`), { ts: tsSec, data });
+      }
+    } catch (err) {
+      clearTimeout(fallbackTimer);
+      if (responded) return;
+
+      const e = err as any;
+      const reason = e.name === 'AbortError' ? `timeout (${timeoutMs} ms)` : e.message;
+      console.warn(`[cache] ${base} → fallback (${reason})`);
+
+      if (latest && Date.now() - latest.ts * 1_000 < ttlMs) {
+        res.setHeader('X-Cache', 'HIT');
+        const cached = await readFileData(path.join(CACHE_SUBDIR, latest.file));
+        return res.json(cached.data);
+      }
+
+      res.setHeader('X-Cache', 'MISS');
+      return res.status(504).json({ error: 'Gateway Timeout and no cache' });
+    }
+  });
+}

defi/src/api2/cache/file-cache.ts

@@ -1,10 +1,10 @@
+import { log, } from '@defillama/sdk';
+import { sliceIntoChunks } from '@defillama/sdk/build/util';
 import fs from 'fs';
 import path from 'path';
 import { METADATA_FILE, PG_CACHE_KEYS } from '../constants';
 import getEnv from '../env';
-import { log, } from '@defillama/sdk'
-import { sliceIntoChunks } from '@defillama/sdk/build/util';
-export { PG_CACHE_KEYS }
+export { PG_CACHE_KEYS };
 
 const CACHE_DIR = getEnv().api2CacheDir;
 export const ROUTES_DATA_DIR = path.join(CACHE_DIR!, 'build')
@@ -231,3 +231,6 @@ export async function storeHistoricalTVLMetadataFile(data: any) {
     i++
   }
 }
+
+export { readFileData, storeData };
+

defi/src/api2/routes/index.ts

@@ -1,28 +1,29 @@
 import * as HyperExpress from "hyper-express";
-import { cache, getLastHourlyRecord, getLastHourlyTokensUsd, protocolHasMisrepresentedTokens, } from "../cache";
-import { readRouteData, } from "../cache/file-cache";
-import sluggify from "../../utils/sluggify";
-import { cachedCraftProtocolV2 } from "../utils/craftProtocolV2";
-import { cachedCraftParentProtocolV2 } from "../utils/craftParentProtocolV2";
-import { get20MinDate } from "../../utils/shared";
-import { getTokensInProtocolsInternal } from "../../getTokenInProtocols";
-import { successResponse, errorResponse, errorWrapper as ew } from "./utils";
+import { getCategoryChartByChainData, getTagChartByChainData } from "../../getCategoryChartByChainData";
+import { getChainChartData } from "../../getChart";
+import { getChainDefaultChartData } from "../../getDefaultChart";
+import { getFormattedChains } from "../../getFormattedChains";
+import { computeInflowsData } from "../../getInflows";
 import { getSimpleChainDatasetInternal } from "../../getSimpleChainDataset";
+import { getTokensInProtocolsInternal } from "../../getTokenInProtocols";
 import craftCsvDataset from "../../storeTvlUtils/craftCsvDataset";
-import { getCurrentUnixTimestamp } from "../../utils/date";
 import { getTweetStats } from "../../twitter/db";
-import { getClosestProtocolItem } from "../db";
+import { getCurrentUnixTimestamp } from "../../utils/date";
 import { hourlyTokensTvl, hourlyUsdTokensTvl } from "../../utils/getLastRecord";
-import { computeInflowsData } from "../../getInflows";
-import { getFormattedChains } from "../../getFormattedChains";
+import { chainNameToIdMap } from "../../utils/normalizeChain";
 import { getR2 } from "../../utils/r2";
-import { getChainChartData } from "../../getChart";
-import { getChainDefaultChartData } from "../../getDefaultChart";
-import { getOverviewFileRoute, getDimensionProtocolFileRoute } from "./dimensions";
+import { get20MinDate } from "../../utils/shared";
+import sluggify from "../../utils/sluggify";
+import { cache, getLastHourlyRecord, getLastHourlyTokensUsd, protocolHasMisrepresentedTokens, } from "../cache";
+import setApiFetchCacheRoute from '../cache/api-fetch-cache';
+import { readRouteData, } from "../cache/file-cache";
+import { getClosestProtocolItem } from "../db";
+import { cachedCraftParentProtocolV2 } from "../utils/craftParentProtocolV2";
+import { cachedCraftProtocolV2 } from "../utils/craftProtocolV2";
 import { getDimensionsMetadata } from "../utils/dimensionsUtils";
-import { chainNameToIdMap } from "../../utils/normalizeChain";
+import { getDimensionProtocolFileRoute, getOverviewFileRoute } from "./dimensions";
 import { setInternalRoutes } from "./internalRoutes";
-import { getCategoryChartByChainData, getTagChartByChainData } from "../../getCategoryChartByChainData";
+import { errorResponse, errorWrapper as ew, successResponse } from "./utils";
 
 /* import { getProtocolUsersHandler } from "../../getProtocolUsers";
 import { getActiveUsers } from "../../getActiveUsers";
@@ -41,7 +42,7 @@ export default function setRoutes(router: HyperExpress.Router, routerBasePath: s
   // router.get("/hourly/:name", (async (req, res) => getProtocolishData(req, res, { dataType: 'protocol', useHourlyData: true, skipAggregatedTvl: false })));  // too expensive to handle here
   // router.get("/config/:chain/:contract", ew(getContractName));  // too many requests to handle here
   // add secret route to delete from PG cache
-
+  setApiFetchCacheRoute(router);
   router.get("/protocol/:name", ew(async (req: any, res: any) => getProtocolishData(req, res, {
     dataType: 'protocol', skipAggregatedTvl: false, useNewChainNames: false, restrictResponseSize: req.query_parameters.restrictResponseSize !== 'false'
   })));