rhedyn/src/cache.js

import fs from "node:fs/promises"
import path from "path"
import { createHash } from "crypto"
import { createReadStream } from "fs"
import stableStringify from "safe-stable-stringify"
import {
  slugifyString,
  checkFilesExist,
  getValueAtPath,
  removeCwd,
  getDeepestPropertiesForKey,
} from "./util.js"

export function hashObject(obj) {
  const str = stableStringify(obj)
  if (!hashCache.has(str)) {
    const hashString = createHash("md5")
      .update(str)
      .digest("hex")
    hashCache.set(str, hashString)
    return hashString
  } else {
    const cachedHashString = hashCache.get(str)
    return cachedHashString
  }
}

const hashCache = new Map()
export async function getFileHash(filePath, algorithm = "md5") {
  return new Promise((resolve, reject) => {
    if (!hashCache.has(filePath)) {
      const hash = createHash(algorithm)
      const stream = createReadStream(filePath)

      stream.on("error", reject)
      stream.on("data", chunk => hash.update(chunk))
      stream.on("end", () => {
        const hashString = hash.digest("hex")
        hashCache.set(filePath, hashString)
        resolve(hashString)
      })
    } else {
      const cachedHashString = hashCache.get(filePath)
      resolve(cachedHashString)
    }
  })
}

async function getFileHashes(pathDeps) {
  return Promise.all(
    Object.keys(pathDeps).map(async filePath => {
      const hash = await getFileHash(filePath)
      if (hash !== pathDeps[filePath]) {
        return Promise.reject({ filePath, hash })
      }

      return Promise.resolve(pathDeps[filePath])
    }),
  )
}

function getStatePropsHash(state, props) {
  const stateValues = props.reduce((depmap, dep) => {
    const value = getValueAtPath(state, dep)
    return { ...depmap, [dep]: value }
  }, {})
  return hashObject(stateValues)
}
export async function checkCache(cacheKey, currentState, opts) {
  const name = slugifyString(cacheKey)
  const existingCacheObject = await readCache(opts.cacheDir, name)
  if (existingCacheObject.exists) {
    const fileChecks = opts.ignoreExisting ? {} : await checkFilesExist(existingCacheObject.content.taskResult.paths, opts.outDir)
    if (!fileChecks.absent?.length) {
      const stateHash = getStatePropsHash(
        currentState,
        existingCacheObject.content.deps.state.props,
      )
      if (stateHash === existingCacheObject.content.deps.state.hash) {
        try {
          await getFileHashes(existingCacheObject.content.deps.paths)
          return { hit: true, taskResult: existingCacheObject.content.taskResult, filePath: existingCacheObject.filePath }
        } catch (e) {
          const updates = {
            deps: {
              paths: [e],
            },
          }
          return { hit: false, reason: `File hash mismatch: ${e.filePath}`, updates, filePath: existingCacheObject.filePath }
        }
      }
      const updates = {
        deps: {
          state: {
            ...existingCacheObject.content.deps.state,
            hash: stateHash,
          },
        },
      }
      const stateValuesList = existingCacheObject.content.deps.state.values
      if (stateValuesList) {
        const mismatchedStateProps = existingCacheObject.content.deps.state.props.filter((stateProp, index) => {
          const currentValue = getValueAtPath(currentState, stateProp)
          const cachedValue = stateValuesList[index]
          return currentValue !== cachedValue
        })
        return { hit: false, reason: `State hash mismatch: ${mismatchedStateProps.join(", ")}`, updates, filePath: existingCacheObject.filePath }
      }
      return { hit: false, reason: "State hash mismatch (no values were found in cache)", updates, filePath: existingCacheObject.filePath }
    }
    if (opts.clean) {
      const outFiles = existingCacheObject.content.taskResult.paths
      await Promise.all(
        outFiles.map(
          async outFile =>
            await fs.rm(path.join(opts.outDir, outFile), { force: true }),
        ),
      )
    }
    return { hit: false, reason: `Missing output file(s): ${fileChecks.absent.join(", ")}`, filePath: existingCacheObject.filePath }
  }
  return { hit: false, reason: `Missing cache file: ${existingCacheObject.filePath}`, filePath: existingCacheObject.filePath }
}

export async function updateCache(
  cacheDir,
  cacheKey,
  pathDeps,
  stateDeps,
  taskResult,
  updates,
  includeStateValues,
) {
  await fs.mkdir(cacheDir, { recursive: true })
  const name = slugifyString(cacheKey)
  const accessedState = getDeepestPropertiesForKey([...stateDeps], "path")
  const deps = {
    paths: [...new Set(removeCwd(pathDeps))],
    state: accessedState.reduce(
      (as, { path, value }) => ({ ...as, [path]: value }),
      {},
    ),
  }
  const statePropsList = Object.keys(deps.state)
  const stateValuesList = Object.values(deps.state)
  const updatesStateHash = updates?.deps?.state?.props || []
  const stateDepsHash =
    JSON.stringify(statePropsList) === JSON.stringify(updatesStateHash)
      ? updates?.deps?.state?.hash
      : hashObject(deps.state)

  const updatesPathsCache =
    updates?.deps?.paths?.reduce(
      (pc, { filePath, hash }) => ({
        ...pc,
        [filePath]: hash,
      }),
      {},
    ) || {}
  const pathsCache = (await Promise.all(
    deps.paths.map(async filePath => {
      const hash = updatesPathsCache[filePath]
        ? updatesPathsCache[filePath]
        : await getFileHash(filePath)
      return {
        hash,
        filePath,
      }
    }),
  )).reduce((pc, { filePath, hash }) => ({ ...pc, [filePath]: hash }), {})
  const cacheObject = {
    deps: {
      state: {
        hash: stateDepsHash,
        props: Object.keys(deps.state),
      },
      paths: pathsCache,
    },
    taskResult,
  }
  if (includeStateValues) {
    cacheObject.deps.state.values = stateValuesList
  }
  return await writeCache(cacheDir, name, cacheObject)
}

async function writeCache(cacheDir, name, cache) {
  if (!cacheDir) {
    return false
  }
  return fs.writeFile(
    path.join(cacheDir, `${name}.json`),
    JSON.stringify(cache),
    "utf8",
  )
}

async function readCache(cacheDir, name) {
  const filePath = path.join(cacheDir, `${name}.json`)
  if (!cacheDir) {
    return { exists: false, filePath }
  }
  try {
    const content = await fs.readFile(
      filePath,
      "utf8",
    )
    return { exists: true, content: JSON.parse(content), filePath }
  } catch (e) {
    return { exists: false, filePath, reason: e }
  }
}