|
|
@@ -24,8 +24,8 @@ function buildOutputPath(outDir, outputDir, pathsToStrip, inputPath, outputFileE
|
|
|
)
|
|
|
}
|
|
|
|
|
|
-function getPathsToStrip(config) {
|
|
|
- return (config.stripPaths || []).map(p => expandTilde(p))
|
|
|
+function getPathsToStrip(jobConfig) {
|
|
|
+ return (jobConfig.stripPaths || []).map(p => expandTilde(p))
|
|
|
}
|
|
|
|
|
|
function paginateItems(items, itemsPerPage) {
|
|
|
@@ -112,20 +112,22 @@ export async function getConfig() {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
-async function runTask({ meta, config, jobId }) {
|
|
|
+async function runTask({ meta, action, jobConfig, actionConfig, jobId }) {
|
|
|
const log = getLogger(
|
|
|
- config.logLevel ? config.logLevel : meta.opts.logLevel,
|
|
|
+ jobConfig.logLevel ? jobConfig.logLevel : meta.opts.logLevel,
|
|
|
jobId,
|
|
|
)
|
|
|
log.trace(`meta: ${JSON.stringify(meta, null, 2)}`)
|
|
|
- log.trace(`config: ${JSON.stringify(config, null, 2)}`)
|
|
|
+ log.trace(`jobConfig: ${JSON.stringify(jobConfig, null, 2)}`)
|
|
|
+ log.trace(`actionConfig: ${JSON.stringify(actionConfig, null, 2)}`)
|
|
|
|
|
|
const stateObject = {
|
|
|
meta,
|
|
|
- config,
|
|
|
+ config: actionConfig,
|
|
|
+ jobConfig,
|
|
|
}
|
|
|
const cache =
|
|
|
- meta.opts.cacheDir && !config.skipCache
|
|
|
+ meta.opts.cacheDir && !jobConfig.skipCache
|
|
|
? await checkCache(jobId, stateObject, meta.opts)
|
|
|
: { disabled: true, reason: "Cache disabled" }
|
|
|
|
|
|
@@ -142,9 +144,9 @@ async function runTask({ meta, config, jobId }) {
|
|
|
const {
|
|
|
detail = {},
|
|
|
paths = [],
|
|
|
- deps: processorDeps,
|
|
|
+ deps: actionDeps,
|
|
|
ref,
|
|
|
- } = await config.processor(state.proxy)
|
|
|
+ } = await action(state.proxy)
|
|
|
|
|
|
const taskResult = {
|
|
|
detail,
|
|
|
@@ -156,17 +158,17 @@ async function runTask({ meta, config, jobId }) {
|
|
|
log.debug(`Wrote ${taskResult.paths.length} files for ${jobId}`)
|
|
|
if (cache && !cache.disabled) {
|
|
|
log.debug(`Updating cache for ${jobId}: ${cache.filePath}`)
|
|
|
- const processorPathDeps = processorDeps?.paths || []
|
|
|
- const processorStateDeps = processorDeps?.state || []
|
|
|
- const configPathDeps = config.deps?.paths || []
|
|
|
- const configStateDeps = config.deps?.state || []
|
|
|
- const stateSelectors = config.stateSelectors || []
|
|
|
+ const actionPathDeps = actionDeps?.paths || []
|
|
|
+ const actionStateDeps = actionDeps?.state || []
|
|
|
+ const configPathDeps = jobConfig.deps?.paths || []
|
|
|
+ const configStateDeps = jobConfig.deps?.state || []
|
|
|
+ const stateSelectors = jobConfig.stateSelectors || []
|
|
|
await updateCache(
|
|
|
meta.opts.cacheDir,
|
|
|
jobId,
|
|
|
removeCwd(
|
|
|
[
|
|
|
- ...configPathDeps, ...processorPathDeps, config?.filePath,
|
|
|
+ ...configPathDeps, ...actionPathDeps, actionConfig?.filePath,
|
|
|
].filter(
|
|
|
item => !!item,
|
|
|
),
|
|
|
@@ -174,7 +176,7 @@ async function runTask({ meta, config, jobId }) {
|
|
|
[
|
|
|
...configStateDeps,
|
|
|
...stateSelectors,
|
|
|
- ...processorStateDeps,
|
|
|
+ ...actionStateDeps,
|
|
|
...(state?.accessed || []),
|
|
|
].filter(item => !!item),
|
|
|
taskResult,
|
|
|
@@ -186,8 +188,8 @@ async function runTask({ meta, config, jobId }) {
|
|
|
return taskResult
|
|
|
}
|
|
|
|
|
|
-function selectFiles(patternsToInclude, config) {
|
|
|
- const pathsToStrip = getPathsToStrip(config)
|
|
|
+function selectFiles(patternsToInclude, jobConfig) {
|
|
|
+ const pathsToStrip = getPathsToStrip(jobConfig)
|
|
|
|
|
|
return async meta => {
|
|
|
const filesToProcess = await readFilesByGlob(patternsToInclude)
|
|
|
@@ -195,10 +197,10 @@ function selectFiles(patternsToInclude, config) {
|
|
|
return filesToProcess.map(filePath => {
|
|
|
const fileOutputPath = buildOutputPath(
|
|
|
meta.opts.outDir,
|
|
|
- config.outputDir,
|
|
|
+ jobConfig.outputDir,
|
|
|
pathsToStrip,
|
|
|
filePath,
|
|
|
- config.outputFileExtension,
|
|
|
+ jobConfig.outputFileExtension,
|
|
|
)
|
|
|
|
|
|
return {
|
|
|
@@ -210,20 +212,23 @@ function selectFiles(patternsToInclude, config) {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
-async function expandFileTask(patternsToInclude, config, meta) {
|
|
|
- const filesToProcess = await selectFiles(patternsToInclude, config)(meta)
|
|
|
+async function expandFileTask(patternsToInclude, task, jobConfig, actionConfig, meta) {
|
|
|
+ const filesToProcess = await selectFiles(patternsToInclude, jobConfig)(meta)
|
|
|
|
|
|
return await Promise.all(
|
|
|
filesToProcess.map(async fileJob => {
|
|
|
- const jobConfig = {
|
|
|
- ...config,
|
|
|
+ const jobActionConfig = {
|
|
|
+ ...actionConfig,
|
|
|
...fileJob,
|
|
|
+ outputFileExtension: jobConfig.outputFileExtension,
|
|
|
}
|
|
|
|
|
|
return runTask({
|
|
|
meta,
|
|
|
- config: jobConfig,
|
|
|
- jobId: `${config.name} @ ${fileJob.filePath}`,
|
|
|
+ action: task.action,
|
|
|
+ jobConfig,
|
|
|
+ actionConfig: jobActionConfig,
|
|
|
+ jobId: `${task.name} @ ${fileJob.filePath}`,
|
|
|
})
|
|
|
}),
|
|
|
)
|
|
|
@@ -250,10 +255,10 @@ function selectState(stateToSelect, meta) {
|
|
|
.flat()
|
|
|
}
|
|
|
|
|
|
-async function expandStateTask(stateToExpand, config, meta) {
|
|
|
+async function expandStateTask(stateToExpand, task, jobConfig, actionConfig, meta) {
|
|
|
const stateToProcess = selectState(stateToExpand, meta)
|
|
|
- const pathsToStrip = getPathsToStrip(config)
|
|
|
- const itemsPerPage = config.itemsPerPage ?? meta.opts.itemsPerPage
|
|
|
+ const pathsToStrip = getPathsToStrip(jobConfig)
|
|
|
+ const itemsPerPage = jobConfig.itemsPerPage ?? meta.opts.itemsPerPage
|
|
|
|
|
|
// Build jobs, potentially with pagination
|
|
|
const jobs = stateToProcess.flatMap((stateJob, index) => {
|
|
|
@@ -270,10 +275,10 @@ async function expandStateTask(stateToExpand, config, meta) {
|
|
|
return pages.map(({ items: pageItems, page, totalPages }) => {
|
|
|
const basePath = buildOutputPath(
|
|
|
meta.opts.outDir,
|
|
|
- config.outputDir,
|
|
|
+ jobConfig.outputDir,
|
|
|
pathsToStrip,
|
|
|
stateJob.key || String(index),
|
|
|
- config.outputFileExtension,
|
|
|
+ jobConfig.outputFileExtension,
|
|
|
)
|
|
|
|
|
|
// For page 2+, modify the output path
|
|
|
@@ -293,7 +298,7 @@ async function expandStateTask(stateToExpand, config, meta) {
|
|
|
...(Array.isArray(stateJob.value)
|
|
|
? { inputs: pageItems }
|
|
|
: { ...stateJob.value?.detail, inputs: pageItems }),
|
|
|
- ...(config.buildFilePath ? { fileOutputPath } : {}),
|
|
|
+ ...(jobConfig.buildFilePath ? { fileOutputPath } : {}),
|
|
|
...(pagination ? { pagination } : {}),
|
|
|
}
|
|
|
|
|
|
@@ -309,75 +314,101 @@ async function expandStateTask(stateToExpand, config, meta) {
|
|
|
|
|
|
return await Promise.all(
|
|
|
jobs.map(async ({ stateJob, page, decorations }) => {
|
|
|
- const jobConfig = {
|
|
|
- ...config,
|
|
|
+ const fileOutputDir = decorations.fileOutputPath
|
|
|
+ ? path.dirname(decorations.fileOutputPath)
|
|
|
+ : undefined
|
|
|
+ const jobActionConfig = {
|
|
|
+ ...actionConfig,
|
|
|
...decorations,
|
|
|
stateKey: stateJob.key,
|
|
|
+ ...(fileOutputDir ? { fileOutputDir } : {}),
|
|
|
+ outputFileExtension: jobConfig.outputFileExtension,
|
|
|
}
|
|
|
const pageInfo = page > 1 ? `:page-${page}` : ""
|
|
|
return runTask({
|
|
|
meta,
|
|
|
- config: jobConfig,
|
|
|
- jobId: `${config.name} @ ${stateJob.property}:${stateJob.index}${pageInfo}`,
|
|
|
+ action: task.action,
|
|
|
+ jobConfig,
|
|
|
+ actionConfig: jobActionConfig,
|
|
|
+ jobId: `${task.name} @ ${stateJob.property}:${stateJob.index}${pageInfo}`,
|
|
|
})
|
|
|
}),
|
|
|
)
|
|
|
}
|
|
|
|
|
|
-export async function expandAndRunTask(meta, config) {
|
|
|
- const includes = meta.opts?.include?.[config.name] || []
|
|
|
- const patternsToInclude = [...(config?.inputFiles || []), ...includes]
|
|
|
+export async function expandAndRunTask(meta, task) {
|
|
|
+ const jobConfig = task.jobConfig || {}
|
|
|
+ const actionConfig = {
|
|
|
+ ...(task.actionConfig || {}),
|
|
|
+ name: task.name,
|
|
|
+ }
|
|
|
+ const includes = meta.opts?.include?.[task.name] || []
|
|
|
+ const patternsToInclude = [...(jobConfig?.inputFiles || []), ...includes]
|
|
|
|
|
|
if (patternsToInclude.length) {
|
|
|
- if (config.expand === false) {
|
|
|
- const inputs = selectFiles(patternsToInclude, config)
|
|
|
- const jobId = config.jobId || config.name
|
|
|
+ if (jobConfig.expand === false) {
|
|
|
+ const inputs = selectFiles(patternsToInclude, jobConfig)
|
|
|
+ const jobId = jobConfig.jobId || task.name
|
|
|
+ const jobActionConfig = {
|
|
|
+ ...actionConfig,
|
|
|
+ inputs,
|
|
|
+ outputFileExtension: jobConfig.outputFileExtension,
|
|
|
+ }
|
|
|
const taskResult = await runTask({
|
|
|
meta,
|
|
|
- config: { ...config, inputs },
|
|
|
+ action: task.action,
|
|
|
+ jobConfig,
|
|
|
+ actionConfig: jobActionConfig,
|
|
|
jobId,
|
|
|
})
|
|
|
return [taskResult]
|
|
|
}
|
|
|
|
|
|
- return expandFileTask(patternsToInclude, config, meta)
|
|
|
+ return expandFileTask(patternsToInclude, task, jobConfig, actionConfig, meta)
|
|
|
}
|
|
|
|
|
|
- if (config.stateSelectors) {
|
|
|
- if (config.expand === false) {
|
|
|
- const pathsToStrip = getPathsToStrip(config)
|
|
|
- const inputs = selectState(config.stateSelectors, meta).map(stateItem => {
|
|
|
+ if (jobConfig.stateSelectors) {
|
|
|
+ if (jobConfig.expand === false) {
|
|
|
+ const pathsToStrip = getPathsToStrip(jobConfig)
|
|
|
+ const inputs = selectState(jobConfig.stateSelectors, meta).map(stateItem => {
|
|
|
return stateItem.value?.detail ?? stateItem.value
|
|
|
})
|
|
|
- const itemsPerPage = config.itemsPerPage
|
|
|
+ const itemsPerPage = jobConfig.itemsPerPage
|
|
|
const shouldPaginate =
|
|
|
Array.isArray(inputs) && itemsPerPage != null && itemsPerPage > 0
|
|
|
const basePath =
|
|
|
- (config.buildFilePath || shouldPaginate)
|
|
|
+ (jobConfig.buildFilePath || shouldPaginate)
|
|
|
? buildOutputPath(
|
|
|
meta.opts.outDir,
|
|
|
- config.outputDir,
|
|
|
+ jobConfig.outputDir,
|
|
|
pathsToStrip,
|
|
|
- config.outputFileName || config.name,
|
|
|
- config.outputFileExtension,
|
|
|
+ jobConfig.outputFileName || task.name,
|
|
|
+ jobConfig.outputFileExtension,
|
|
|
)
|
|
|
: null
|
|
|
|
|
|
if (!shouldPaginate) {
|
|
|
const decorations = {
|
|
|
- ...(config.buildFilePath
|
|
|
+ ...(jobConfig.buildFilePath
|
|
|
? { fileOutputPath: basePath }
|
|
|
: {}),
|
|
|
}
|
|
|
- const jobConfig = {
|
|
|
- ...config,
|
|
|
+ const fileOutputDir = decorations.fileOutputPath
|
|
|
+ ? path.dirname(decorations.fileOutputPath)
|
|
|
+ : undefined
|
|
|
+ const jobActionConfig = {
|
|
|
+ ...actionConfig,
|
|
|
...decorations,
|
|
|
inputs,
|
|
|
+ ...(fileOutputDir ? { fileOutputDir } : {}),
|
|
|
+ outputFileExtension: jobConfig.outputFileExtension,
|
|
|
}
|
|
|
- const jobId = config.jobId || config.name
|
|
|
+ const jobId = jobConfig.jobId || task.name
|
|
|
const taskResult = await runTask({
|
|
|
meta,
|
|
|
- config: { ...jobConfig },
|
|
|
+ action: task.action,
|
|
|
+ jobConfig,
|
|
|
+ actionConfig: jobActionConfig,
|
|
|
jobId,
|
|
|
})
|
|
|
return [taskResult]
|
|
|
@@ -386,19 +417,26 @@ export async function expandAndRunTask(meta, config) {
|
|
|
const pages = paginateItems(inputs, itemsPerPage)
|
|
|
if (pages.length <= 1) {
|
|
|
const decorations = {
|
|
|
- ...(config.buildFilePath
|
|
|
+ ...(jobConfig.buildFilePath
|
|
|
? { fileOutputPath: basePath }
|
|
|
: {}),
|
|
|
}
|
|
|
- const jobConfig = {
|
|
|
- ...config,
|
|
|
+ const fileOutputDir = decorations.fileOutputPath
|
|
|
+ ? path.dirname(decorations.fileOutputPath)
|
|
|
+ : undefined
|
|
|
+ const jobActionConfig = {
|
|
|
+ ...actionConfig,
|
|
|
...decorations,
|
|
|
inputs,
|
|
|
+ ...(fileOutputDir ? { fileOutputDir } : {}),
|
|
|
+ outputFileExtension: jobConfig.outputFileExtension,
|
|
|
}
|
|
|
- const jobId = config.jobId || config.name
|
|
|
+ const jobId = jobConfig.jobId || task.name
|
|
|
const taskResult = await runTask({
|
|
|
meta,
|
|
|
- config: { ...jobConfig },
|
|
|
+ action: task.action,
|
|
|
+ jobConfig,
|
|
|
+ actionConfig: jobActionConfig,
|
|
|
jobId,
|
|
|
})
|
|
|
return [taskResult]
|
|
|
@@ -407,7 +445,7 @@ export async function expandAndRunTask(meta, config) {
|
|
|
const basePathForPagination = basePath
|
|
|
? basePath.replace(meta.opts.outDir, "/")
|
|
|
: null
|
|
|
- const baseJobId = config.jobId || config.name
|
|
|
+ const baseJobId = jobConfig.jobId || task.name
|
|
|
return await Promise.all(
|
|
|
pages.map(({ items: pageItems, page, totalPages }) => {
|
|
|
const fileOutputPath = basePath
|
|
|
@@ -424,31 +462,44 @@ export async function expandAndRunTask(meta, config) {
|
|
|
? buildPaginationMeta(page, totalPages, basePathForPagination)
|
|
|
: null
|
|
|
const decorations = {
|
|
|
- ...(config.buildFilePath && fileOutputPath
|
|
|
+ ...(jobConfig.buildFilePath && fileOutputPath
|
|
|
? { fileOutputPath }
|
|
|
: {}),
|
|
|
...(pagination ? { pagination } : {}),
|
|
|
}
|
|
|
- const jobConfig = {
|
|
|
- ...config,
|
|
|
+ const fileOutputDir = fileOutputPath
|
|
|
+ ? path.dirname(fileOutputPath)
|
|
|
+ : undefined
|
|
|
+ const jobActionConfig = {
|
|
|
+ ...actionConfig,
|
|
|
...decorations,
|
|
|
inputs: pageItems,
|
|
|
+ ...(fileOutputDir ? { fileOutputDir } : {}),
|
|
|
+ outputFileExtension: jobConfig.outputFileExtension,
|
|
|
}
|
|
|
const pageInfo = page > 1 ? `:page-${page}` : ""
|
|
|
return runTask({
|
|
|
meta,
|
|
|
- config: { ...jobConfig },
|
|
|
+ action: task.action,
|
|
|
+ jobConfig,
|
|
|
+ actionConfig: jobActionConfig,
|
|
|
jobId: `${baseJobId}${pageInfo}`,
|
|
|
})
|
|
|
}),
|
|
|
)
|
|
|
}
|
|
|
|
|
|
- return expandStateTask(config.stateSelectors, config, meta)
|
|
|
+ return expandStateTask(jobConfig.stateSelectors, task, jobConfig, actionConfig, meta)
|
|
|
}
|
|
|
|
|
|
- const jobId = config.jobId || config.name
|
|
|
- const taskResult = await runTask({ meta, config, jobId })
|
|
|
+ const jobId = jobConfig.jobId || task.name
|
|
|
+ const taskResult = await runTask({
|
|
|
+ meta,
|
|
|
+ action: task.action,
|
|
|
+ jobConfig,
|
|
|
+ actionConfig,
|
|
|
+ jobId,
|
|
|
+ })
|
|
|
return [taskResult]
|
|
|
}
|
|
|
|
|
|
@@ -459,9 +510,10 @@ export async function processTask(meta, task) {
|
|
|
const cached = taskResult.filter(taskResult => taskResult.fromCache)
|
|
|
const processed = taskResult.filter(taskResult => !taskResult.fromCache)
|
|
|
const resourcesWithRef = taskResult.filter(tResult => tResult.ref)
|
|
|
+ const jobConfig = task.jobConfig || {}
|
|
|
const hasExpandableInputs =
|
|
|
- (task.inputFiles?.length || task.stateSelectors?.length) &&
|
|
|
- task.expand !== false
|
|
|
+ (jobConfig.inputFiles?.length || jobConfig.stateSelectors?.length) &&
|
|
|
+ jobConfig.expand !== false
|
|
|
const shouldCollapseResources =
|
|
|
resourcesWithRef.length === 1 &&
|
|
|
taskResult.length === 1 &&
|