fix: contain automation callback failures
This commit is contained in:
codex
@@ -105,11 +105,17 @@ internal sealed record FirstApplyAutomationEvidence
|
||||
public bool? PermitAccepted { get; init; }
|
||||
public int? IdleCallbackCount { get; init; }
|
||||
public int? CoordinatorStartCount { get; init; }
|
||||
// Local test evidence only. This is intentionally never copied into Git
|
||||
// evidence because exception stacks may contain machine-specific paths.
|
||||
public string? ExceptionText { get; init; }
|
||||
public string? PreviousState { get; init; }
|
||||
public string? CurrentState { get; init; }
|
||||
public IReadOnlyList<string> StateTransitions { get; init; } = Array.Empty<string>();
|
||||
}
|
||||
|
||||
internal sealed class FirstLimitedApplyAutomationCoordinator
|
||||
{
|
||||
private enum CoordinatorState { Created, Starting, Running, Completed, Failed, ExitQueued }
|
||||
private sealed record PxpResult(string? PlanId, IReadOnlyList<string> Candidates, string? View, int Create, int Skip, bool? PermitAccepted);
|
||||
private static string? selectedTargetSnapshotId;
|
||||
private static readonly List<PxpResult> pxpResults = [];
|
||||
@@ -121,21 +127,27 @@ internal sealed class FirstLimitedApplyAutomationCoordinator
|
||||
if (string.Equals(Environment.GetEnvironmentVariable("P_REC_ANNOT_AUTOMATION"), "FirstLimitedApplyRuntimeTest", StringComparison.Ordinal))
|
||||
pxpResults.Add(new(planId, candidates, view, create, skip, permitAccepted));
|
||||
}
|
||||
private FirstApplyAutomationConfiguration config = null!;
|
||||
private Document document = null!;
|
||||
private FirstApplyAutomationConfiguration? config;
|
||||
private Document? document;
|
||||
private readonly List<string> completed = [];
|
||||
private int step;
|
||||
private int? before, after1, after2, afterUndo;
|
||||
private AutomationEntityCounts? baseline, counts1, counts2, countsUndo;
|
||||
private readonly List<string> transitions = ["Created"];
|
||||
private CoordinatorState state = CoordinatorState.Created;
|
||||
private CoordinatorState previousState = CoordinatorState.Created;
|
||||
private bool documentHandlersSubscribed;
|
||||
private bool lifecycleHandlersSubscribed;
|
||||
private bool exitQueued;
|
||||
private bool exitHandlerSubscribed;
|
||||
private string? exceptionText;
|
||||
|
||||
public void Start()
|
||||
{
|
||||
try
|
||||
{
|
||||
Transition(CoordinatorState.Starting, "PostStartupReady");
|
||||
config = ReadConfiguration();
|
||||
transitions.Add("PostStartupReady");
|
||||
if (string.Equals(config.Phase, "LifecycleProbe", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
CompleteLifecycleProbe();
|
||||
@@ -157,8 +169,11 @@ internal sealed class FirstLimitedApplyAutomationCoordinator
|
||||
document.CommandCancelled += OnCommandCancelled;
|
||||
AcadApplication.DocumentManager.DocumentActivated += OnDocumentActivated;
|
||||
AcadApplication.Idle += OnIdle;
|
||||
documentHandlersSubscribed = true;
|
||||
lifecycleHandlersSubscribed = true;
|
||||
Transition(CoordinatorState.Running, "AutomationRunning");
|
||||
}
|
||||
catch (System.Exception ex) { Complete(false, "AUTOMATION_START_FAILED:" + ex.GetType().Name); }
|
||||
catch (System.Exception ex) { FailFromCallback("Start", ex, "AUTOMATION_START_FAILED"); }
|
||||
}
|
||||
|
||||
private void CompleteLifecycleProbe()
|
||||
@@ -168,18 +183,21 @@ internal sealed class FirstLimitedApplyAutomationCoordinator
|
||||
RequestControlledExit();
|
||||
}
|
||||
|
||||
private void OnIdle(object? sender, EventArgs e)
|
||||
{
|
||||
BeginContextVerification();
|
||||
}
|
||||
private void OnIdle(object? sender, EventArgs e) => RunCallback("Idle", BeginContextVerification);
|
||||
private void OnDocumentActivated(object sender, DocumentCollectionEventArgs e)
|
||||
{
|
||||
if (string.Equals(e.Document.Name, document.Name, StringComparison.OrdinalIgnoreCase)) BeginContextVerification();
|
||||
RunCallback("DocumentActivated", () =>
|
||||
{
|
||||
if (document is null || e.Document is null) return;
|
||||
if (string.Equals(e.Document.Name, document.Name, StringComparison.OrdinalIgnoreCase)) BeginContextVerification();
|
||||
});
|
||||
}
|
||||
private void BeginContextVerification()
|
||||
{
|
||||
if (IsTerminal || document is null) return;
|
||||
AcadApplication.Idle -= OnIdle;
|
||||
AcadApplication.DocumentManager.DocumentActivated -= OnDocumentActivated;
|
||||
lifecycleHandlersSubscribed = false;
|
||||
try
|
||||
{
|
||||
transitions.Add("ContextVerificationStarted");
|
||||
@@ -194,31 +212,36 @@ internal sealed class FirstLimitedApplyAutomationCoordinator
|
||||
before = Dbmod(); baseline = Counts();
|
||||
Queue("PXP-OA");
|
||||
}
|
||||
catch (System.Exception ex) { Complete(false, "AUTOMATION_CONTEXT_FAILED:" + ex.GetType().Name); }
|
||||
catch (System.Exception ex) { FailFromCallback("ContextVerification", ex, "AUTOMATION_CONTEXT_FAILED"); }
|
||||
}
|
||||
|
||||
private void Queue(string command)
|
||||
{
|
||||
if (IsTerminal || document is null) return;
|
||||
transitions.Add("CommandQueued:" + command);
|
||||
document.SendStringToExecute(command + " ", true, false, false);
|
||||
}
|
||||
private void OnCommandEnded(object sender, CommandEventArgs e)
|
||||
{
|
||||
var name = e.GlobalCommandName?.Trim().TrimStart('.', '_').ToUpperInvariant();
|
||||
if (name is not ("PXP-OA" or "UNDO" or "U")) return;
|
||||
completed.Add(name);
|
||||
transitions.Add("CommandCompleted:" + name);
|
||||
if (step++ == 0) { after1 = Dbmod(); counts1 = Counts(); Queue("PXP-OA"); return; }
|
||||
if (step == 2 && string.Equals(config.Phase, "Apply", StringComparison.OrdinalIgnoreCase)) { after2 = Dbmod(); counts2 = Counts(); Queue("_.UNDO _1"); return; }
|
||||
if (string.Equals(config.Phase, "Apply", StringComparison.OrdinalIgnoreCase)) { afterUndo = Dbmod(); countsUndo = Counts(); Complete(true, null); }
|
||||
else { after2 = Dbmod(); counts2 = Counts(); Complete(true, null); }
|
||||
RunCallback("CommandEnded", () =>
|
||||
{
|
||||
var name = e.GlobalCommandName?.Trim().TrimStart('.', '_').ToUpperInvariant();
|
||||
if (name is not ("PXP-OA" or "UNDO" or "U")) return;
|
||||
completed.Add(name);
|
||||
transitions.Add("CommandCompleted:" + name);
|
||||
if (step++ == 0) { after1 = Dbmod(); counts1 = Counts(); Queue("PXP-OA"); return; }
|
||||
if (step == 2 && string.Equals(config?.Phase, "Apply", StringComparison.OrdinalIgnoreCase)) { after2 = Dbmod(); counts2 = Counts(); Queue("_.UNDO _1"); return; }
|
||||
if (string.Equals(config?.Phase, "Apply", StringComparison.OrdinalIgnoreCase)) { afterUndo = Dbmod(); countsUndo = Counts(); Complete(true, null); }
|
||||
else { after2 = Dbmod(); counts2 = Counts(); Complete(true, null); }
|
||||
});
|
||||
}
|
||||
private void OnCommandFailed(object sender, CommandEventArgs e) => Complete(false, "AUTOMATION_COMMAND_FAILED:" + e.GlobalCommandName);
|
||||
private void OnCommandCancelled(object sender, CommandEventArgs e) => Complete(false, "AUTOMATION_COMMAND_CANCELLED:" + e.GlobalCommandName);
|
||||
private void OnCommandFailed(object sender, CommandEventArgs e) => RunCallback("CommandFailed", () => Complete(false, "AUTOMATION_COMMAND_FAILED:" + e.GlobalCommandName));
|
||||
private void OnCommandCancelled(object sender, CommandEventArgs e) => RunCallback("CommandCancelled", () => Complete(false, "AUTOMATION_COMMAND_CANCELLED:" + e.GlobalCommandName));
|
||||
|
||||
private int Dbmod() => Convert.ToInt32(AcadApplication.GetSystemVariable("DBMOD"));
|
||||
private AutomationEntityCounts Counts()
|
||||
{
|
||||
if (document is null) throw new InvalidOperationException("AUTOMATION_DOCUMENT_UNAVAILABLE");
|
||||
using var tx = document.Database.TransactionManager.StartOpenCloseTransaction();
|
||||
var table = (BlockTable)tx.GetObject(document.Database.BlockTableId, OpenMode.ForRead);
|
||||
var paper = (BlockTableRecord)tx.GetObject(table[BlockTableRecord.PaperSpace], OpenMode.ForRead);
|
||||
@@ -250,16 +273,13 @@ internal sealed class FirstLimitedApplyAutomationCoordinator
|
||||
}
|
||||
private void Complete(bool success, string? error)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (document is not null) { document.CommandEnded -= OnCommandEnded; document.CommandFailed -= OnCommandFailed; document.CommandCancelled -= OnCommandCancelled; }
|
||||
AcadApplication.DocumentManager.DocumentActivated -= OnDocumentActivated;
|
||||
WriteEvidence(success, error);
|
||||
if (document is not null) document.CloseAndDiscard();
|
||||
RequestControlledExit();
|
||||
}
|
||||
catch { }
|
||||
finally { selectedTargetSnapshotId = null; pxpResults.Clear(); }
|
||||
if (IsTerminal) return;
|
||||
Transition(success ? CoordinatorState.Completed : CoordinatorState.Failed, success ? "Completed" : "Failed");
|
||||
UnsubscribeHandlers();
|
||||
TryWriteEvidence(success, error);
|
||||
selectedTargetSnapshotId = null;
|
||||
pxpResults.Clear();
|
||||
RequestControlledExit();
|
||||
}
|
||||
|
||||
// AutoCAD's direct Application.Quit can leave a startup-script host alive.
|
||||
@@ -269,32 +289,81 @@ internal sealed class FirstLimitedApplyAutomationCoordinator
|
||||
{
|
||||
if (exitQueued) return;
|
||||
exitQueued = true;
|
||||
transitions.Add("ExitQueued");
|
||||
Transition(CoordinatorState.ExitQueued, "ExitQueued");
|
||||
AcadApplication.Idle += OnExitIdle;
|
||||
exitHandlerSubscribed = true;
|
||||
}
|
||||
|
||||
private void OnExitIdle(object? sender, EventArgs args)
|
||||
{
|
||||
AcadApplication.Idle -= OnExitIdle;
|
||||
try
|
||||
RunCallback("ControlledExitIdle", () =>
|
||||
{
|
||||
if (exitHandlerSubscribed) AcadApplication.Idle -= OnExitIdle;
|
||||
exitHandlerSubscribed = false;
|
||||
// Do not close a cached automation document and then reuse it. Resolve
|
||||
// the currently-live document only at the final quiescent callback.
|
||||
var active = AcadApplication.DocumentManager.MdiActiveDocument;
|
||||
if (active is null) { AcadApplication.Quit(); return; }
|
||||
transitions.Add("NativeQuitDispatched");
|
||||
active.SendStringToExecute("_.QUIT _N ", true, false, false);
|
||||
}
|
||||
catch (System.Exception exception)
|
||||
});
|
||||
}
|
||||
|
||||
private bool IsTerminal => state is CoordinatorState.Completed or CoordinatorState.Failed or CoordinatorState.ExitQueued;
|
||||
|
||||
private void Transition(CoordinatorState next, string transition)
|
||||
{
|
||||
previousState = state;
|
||||
state = next;
|
||||
transitions.Add(transition);
|
||||
}
|
||||
|
||||
private void RunCallback(string callback, Action action)
|
||||
{
|
||||
if (IsTerminal && callback != "ControlledExitIdle") return;
|
||||
try { action(); }
|
||||
catch (System.Exception exception) { FailFromCallback(callback, exception, "AUTOMATION_CALLBACK_FAILED"); }
|
||||
}
|
||||
|
||||
private void FailFromCallback(string callback, System.Exception exception, string code)
|
||||
{
|
||||
exceptionText = exception.ToString();
|
||||
RuntimeDiagnostics.Write("AutomationCallbackFailed", "ERROR", resultStatus: code + ":" + callback, exception: exception);
|
||||
if (!IsTerminal) Complete(false, code + ":" + callback + ":" + exception.GetType().Name);
|
||||
else
|
||||
{
|
||||
RuntimeDiagnostics.Write("AutomationControlledExitFailed", "ERROR", resultStatus: "AUTOMATION_CONTROLLED_EXIT_FAILED", exception: exception);
|
||||
AcadApplication.Quit();
|
||||
UnsubscribeHandlers();
|
||||
TryWriteEvidence(false, code + ":" + callback + ":" + exception.GetType().Name);
|
||||
}
|
||||
}
|
||||
|
||||
private void UnsubscribeHandlers()
|
||||
{
|
||||
if (documentHandlersSubscribed && document is not null)
|
||||
{
|
||||
try { document.CommandEnded -= OnCommandEnded; document.CommandFailed -= OnCommandFailed; document.CommandCancelled -= OnCommandCancelled; }
|
||||
catch (System.Exception exception) { RuntimeDiagnostics.Write("AutomationHandlerUnsubscribeFailed", "ERROR", resultStatus: "AUTOMATION_UNSUBSCRIBE_FAILED", exception: exception); }
|
||||
}
|
||||
documentHandlersSubscribed = false;
|
||||
if (lifecycleHandlersSubscribed)
|
||||
{
|
||||
try { AcadApplication.DocumentManager.DocumentActivated -= OnDocumentActivated; AcadApplication.Idle -= OnIdle; }
|
||||
catch (System.Exception exception) { RuntimeDiagnostics.Write("AutomationHandlerUnsubscribeFailed", "ERROR", resultStatus: "AUTOMATION_UNSUBSCRIBE_FAILED", exception: exception); }
|
||||
}
|
||||
lifecycleHandlersSubscribed = false;
|
||||
}
|
||||
|
||||
private void TryWriteEvidence(bool success, string? error)
|
||||
{
|
||||
try { WriteEvidence(success, error); }
|
||||
catch (System.Exception exception) { RuntimeDiagnostics.Write("AutomationEvidenceWriteFailed", "ERROR", resultStatus: "AUTOMATION_EVIDENCE_WRITE_FAILED", exception: exception); }
|
||||
}
|
||||
|
||||
private void WriteEvidence(bool success, string? error)
|
||||
{
|
||||
if (config is null) return;
|
||||
var first = pxpResults.ElementAtOrDefault(0); var second = pxpResults.ElementAtOrDefault(1);
|
||||
var evidence = new FirstApplyAutomationEvidence { Phase = config.Phase, Completed = success, ErrorCode = error, DbmodBefore = before, DbmodAfterCommand1 = after1, DbmodAfterCommand2 = after2, DbmodAfterUndo = afterUndo, Baseline = baseline, AfterCommand1 = counts1, AfterCommand2 = counts2, AfterUndo = countsUndo, CompletedCommands = completed, PlanIdRun1 = first?.PlanId, PlanIdRun2 = second?.PlanId, PlanIdsEqual = first is not null && second is not null ? string.Equals(first.PlanId, second.PlanId, StringComparison.Ordinal) : null, EligibleCandidateCount = first?.Candidates.Count, SelectedCandidateHash = selectedTargetSnapshotId, ViewHash = first?.View, CreateRun1 = first?.Create, CreateRun2 = second?.Create, SkipRun2 = second?.Skip, PermitAccepted = first?.PermitAccepted, IdleCallbackCount = FirstLimitedApplyAutomationBootstrap.IdleCallbackCount, CoordinatorStartCount = FirstLimitedApplyAutomationBootstrap.CoordinatorStartCount, StateTransitions = transitions };
|
||||
var evidence = new FirstApplyAutomationEvidence { Phase = config.Phase, Completed = success, ErrorCode = error, DbmodBefore = before, DbmodAfterCommand1 = after1, DbmodAfterCommand2 = after2, DbmodAfterUndo = afterUndo, Baseline = baseline, AfterCommand1 = counts1, AfterCommand2 = counts2, AfterUndo = countsUndo, CompletedCommands = completed, PlanIdRun1 = first?.PlanId, PlanIdRun2 = second?.PlanId, PlanIdsEqual = first is not null && second is not null ? string.Equals(first.PlanId, second.PlanId, StringComparison.Ordinal) : null, EligibleCandidateCount = first?.Candidates.Count, SelectedCandidateHash = selectedTargetSnapshotId, ViewHash = first?.View, CreateRun1 = first?.Create, CreateRun2 = second?.Create, SkipRun2 = second?.Skip, PermitAccepted = first?.PermitAccepted, IdleCallbackCount = FirstLimitedApplyAutomationBootstrap.IdleCallbackCount, CoordinatorStartCount = FirstLimitedApplyAutomationBootstrap.CoordinatorStartCount, ExceptionText = exceptionText, PreviousState = previousState.ToString(), CurrentState = state.ToString(), StateTransitions = transitions };
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(config.EvidencePath)!);
|
||||
File.WriteAllText(config.EvidencePath, JsonSerializer.Serialize(evidence));
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user