Skip to content

Batch and deduplicate action resolution across composite depths #4296

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
stefanpenner wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Batch and deduplicate action resolution across composite depths #4296

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5bcd72f
Batch and deduplicate action resolution across composite depths
Mar 12, 2026
9bea8b0
Merge branch 'main' into batch-action-resolution-optimization
stefanpenner Mar 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 70 additions & 10 deletions src/Runner.Worker/ActionManager.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,9 @@ public sealed class ActionManager : RunnerService, IActionManager
PreStepTracker = new Dictionary<Guid, IActionRunner>()
};
var containerSetupSteps = new List<JobExtensionRunner>();
// Stack-local cache: same action (owner/repo@ref) is resolved only once,
// even if it appears at multiple depths in a composite tree.
var resolvedDownloadInfos = new Dictionary<string, WebApi.ActionDownloadInfo>(StringComparer.OrdinalIgnoreCase);
var depth = 0;
// We are running at the start of a job
if (rootStepId == default(Guid))
Expand All @@ -105,7 +108,7 @@ public sealed class ActionManager : RunnerService, IActionManager
PrepareActionsState result = new PrepareActionsState();
try
{
result = await PrepareActionsRecursiveAsync(executionContext, state, actions, depth, rootStepId);
result = await PrepareActionsRecursiveAsync(executionContext, state, actions, resolvedDownloadInfos, depth, rootStepId);
}
catch (FailedToResolveActionDownloadInfoException ex)
{
Expand Down Expand Up @@ -161,13 +164,14 @@ public sealed class ActionManager : RunnerService, IActionManager
return new PrepareResult(containerSetupSteps, result.PreStepTracker);
}

private async Task<PrepareActionsState> PrepareActionsRecursiveAsync(IExecutionContext executionContext, PrepareActionsState state, IEnumerable<Pipelines.ActionStep> actions, Int32 depth = 0, Guid parentStepId = default(Guid))
private async Task<PrepareActionsState> PrepareActionsRecursiveAsync(IExecutionContext executionContext, PrepareActionsState state, IEnumerable<Pipelines.ActionStep> actions, Dictionary<string, WebApi.ActionDownloadInfo> resolvedDownloadInfos, Int32 depth = 0, Guid parentStepId = default(Guid))
{
ArgUtil.NotNull(executionContext, nameof(executionContext));
if (depth > Constants.CompositeActionsMaxDepth)
{
throw new Exception($"Composite action depth exceeded max depth {Constants.CompositeActionsMaxDepth}");
}

var repositoryActions = new List<Pipelines.ActionStep>();

foreach (var action in actions)
Expand Down Expand Up @@ -195,27 +199,29 @@ public sealed class ActionManager : RunnerService, IActionManager

if (repositoryActions.Count > 0)
{
// Get the download info
var downloadInfos = await GetDownloadInfoAsync(executionContext, repositoryActions);
// Resolve download info, skipping any actions already cached.
await ResolveNewActionsAsync(executionContext, repositoryActions, resolvedDownloadInfos);

// Download each action
// Download each action.
foreach (var action in repositoryActions)
{
var lookupKey = GetDownloadInfoLookupKey(action);
if (string.IsNullOrEmpty(lookupKey))
{
continue;
}

if (!downloadInfos.TryGetValue(lookupKey, out var downloadInfo))
if (!resolvedDownloadInfos.TryGetValue(lookupKey, out var downloadInfo))
{
throw new Exception($"Missing download info for {lookupKey}");
}

await DownloadRepositoryActionAsync(executionContext, downloadInfo);
}

// More preparation based on content in the repository (action.yml)
// Parse action.yml and collect composite sub-actions for batched
// resolution below. Pre/post step registration is deferred until
// after recursion so that HasPre/HasPost reflect the full subtree.
var nextLevel = new List<(Pipelines.ActionStep action, Guid parentId)>();

foreach (var action in repositoryActions)
{
var setupInfo = PrepareRepositoryActionAsync(executionContext, action);
Expand Down Expand Up @@ -247,8 +253,35 @@ public sealed class ActionManager : RunnerService, IActionManager
}
else if (setupInfo != null && setupInfo.Steps != null && setupInfo.Steps.Count > 0)
{
state = await PrepareActionsRecursiveAsync(executionContext, state, setupInfo.Steps, depth + 1, action.Id);
foreach (var step in setupInfo.Steps)
{
nextLevel.Add((step, action.Id));
}
}
}

// Resolve all next-level sub-actions in one batch API call,
// then recurse per parent (which hits the cache, not the API).
if (nextLevel.Count > 0)
{
var nextLevelRepoActions = nextLevel
.Where(x => x.action.Reference.Type == Pipelines.ActionSourceType.Repository)
.Select(x => x.action)
.ToList();
await ResolveNewActionsAsync(executionContext, nextLevelRepoActions, resolvedDownloadInfos);

foreach (var group in nextLevel.GroupBy(x => x.parentId))
{
var groupActions = group.Select(x => x.action).ToList();
state = await PrepareActionsRecursiveAsync(executionContext, state, groupActions, resolvedDownloadInfos, depth + 1, group.Key);
}
}

// Register pre/post steps after recursion so that HasPre/HasPost
// are correct (they depend on _cachedEmbeddedPreSteps/PostSteps
// being populated by the recursive calls above).
foreach (var action in repositoryActions)
{
var repoAction = action.Reference as Pipelines.RepositoryPathReference;
if (repoAction.RepositoryType != Pipelines.PipelineConstants.SelfAlias)
{
Expand Down Expand Up @@ -754,6 +787,33 @@ private async Task BuildActionContainerAsync(IExecutionContext executionContext,
return actionDownloadInfos.Actions;
}

/// <summary>
/// Only resolves actions not already in resolvedDownloadInfos.
/// Results are cached for reuse at deeper recursion levels.
/// </summary>
private async Task ResolveNewActionsAsync(IExecutionContext executionContext, List<Pipelines.ActionStep> actions, Dictionary<string, WebApi.ActionDownloadInfo> resolvedDownloadInfos)
{
var actionsToResolve = new List<Pipelines.ActionStep>();
var pendingKeys = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var action in actions)
{
var lookupKey = GetDownloadInfoLookupKey(action);
if (!string.IsNullOrEmpty(lookupKey) && !resolvedDownloadInfos.ContainsKey(lookupKey) && pendingKeys.Add(lookupKey))
{
actionsToResolve.Add(action);
}
}

if (actionsToResolve.Count > 0)
{
var downloadInfos = await GetDownloadInfoAsync(executionContext, actionsToResolve);
foreach (var kvp in downloadInfos)
{
resolvedDownloadInfos[kvp.Key] = kvp.Value;
}
}
}

private async Task DownloadRepositoryActionAsync(IExecutionContext executionContext, WebApi.ActionDownloadInfo downloadInfo)
{
Trace.Entering();
Expand Down
Loading