From f7fa0363e3a7500ad827ceb67876fde56eb0ea6a Mon Sep 17 00:00:00 2001 From: codex Date: Thu, 30 Jul 2026 15:24:51 +0300 Subject: [PATCH] feat: add learned document template workflow --- DocumentAutomation/docs/template-learning.md | 14 ++ .../TemplateModels.cs | 153 ++++++++++++++++++ .../src/DocxWorkbench.App/MainForm.cs | 72 +++++---- .../src/DocxWorkbench.Core/Contracts.cs | 21 +++ .../TemplateLearningService.cs | 73 +++++++++ .../TemplateDocumentGenerator.cs | 49 ++++++ .../WordDocumentNormalizer.cs | 42 +++++ .../WordDocumentService.cs | 4 +- ...DocumentAutomation.IntegrationTests.csproj | 2 +- .../WordComIntegrationTests.cs | 13 ++ .../DocxWorkbench.Tests.csproj | 2 +- .../TemplateLearningTests.cs | 25 +++ 12 files changed, 439 insertions(+), 31 deletions(-) create mode 100644 DocumentAutomation/docs/template-learning.md create mode 100644 DocumentAutomation/src/DocumentAutomation.Shared/TemplateModels.cs create mode 100644 DocumentAutomation/src/DocxWorkbench.OpenXml/TemplateLearningService.cs create mode 100644 DocumentAutomation/src/DocxWorkbench.WordInterop/TemplateDocumentGenerator.cs create mode 100644 DocumentAutomation/src/DocxWorkbench.WordInterop/WordDocumentNormalizer.cs create mode 100644 DocumentAutomation/tests/DocxWorkbench.Tests/TemplateLearningTests.cs diff --git a/DocumentAutomation/docs/template-learning.md b/DocumentAutomation/docs/template-learning.md new file mode 100644 index 0000000..d1da625 --- /dev/null +++ b/DocumentAutomation/docs/template-learning.md @@ -0,0 +1,14 @@ +# Learned document templates + +DOCX Workbench supports two deterministic workflows: + +1. `TemplateLearningService` normalizes a `.doc` or copies a `.docx` into a temporary working directory, inspects it, adds a safe `{{DOC_BODY}}` insertion marker to the working copy, and writes a portable `.dtemplate` ZIP package. +2. `TemplateDocumentGenerator` opens the package, replaces explicitly supplied scalar tokens, imports a `.doc`, `.docx`, or `.txt` body through the existing main-story importer, and creates DOCX, PDF, generation, and validation outputs. + +The package contains `manifest.json`, `template.docx`, source metadata, page archetypes, regions, slots, field bindings, mapping suggestions, and a validation profile. It contains no original source binary unless it is the normalized template copy. + +All Word automation is late-bound and serialized through the existing STA queue. Binary `.doc` sources are opened read-only, macros are disabled through Word automation security, and the source SHA-256 is checked before and after normalization. The source body’s headers, footers, section properties, and page setup are excluded by `DocxBodyImporter`. + +The current UI exposes three modes: template learning, document generation, and the existing quick `DOC_BODY` path. Page archetypes and slot proposals are deterministic suggestions and require later user confirmation; Ollama is not required for the workflow. + +Known limitation: the first slice stores the learned structural model and default slots, but does not yet provide a pixel-level page editor or automatic extraction of every scalar field from an arbitrary engineering drawing. Those operations remain explicit user mapping work. diff --git a/DocumentAutomation/src/DocumentAutomation.Shared/TemplateModels.cs b/DocumentAutomation/src/DocumentAutomation.Shared/TemplateModels.cs new file mode 100644 index 0000000..ba48ff5 --- /dev/null +++ b/DocumentAutomation/src/DocumentAutomation.Shared/TemplateModels.cs @@ -0,0 +1,153 @@ +namespace DocumentAutomation.Shared; + +public enum TemplateSlotKind { PlainText, RichText, MainBody, Table, RepeatingTableRows, Image, Signature, Date, DocumentField } +public enum TemplateRegionKind { Fixed, Variable, Repeating, Ignored } + +public sealed record TemplateSlotOccurrence +{ + public required string Location { get; init; } + public int? Page { get; init; } + public string? Story { get; init; } + public string? XPath { get; init; } +} + +public sealed record TemplateSlot +{ + public required string Id { get; init; } + public required string Name { get; init; } + public required TemplateSlotKind Kind { get; init; } + public required string ContentControlTag { get; init; } + public bool IsRequired { get; init; } + public bool IsRepeating { get; init; } + public string? PageArchetypeId { get; init; } + public string? RepeatGroup { get; init; } + public required IReadOnlyList Occurrences { get; init; } +} + +public sealed record FixedRegion +{ + public required string Id { get; init; } + public required string Name { get; init; } + public required TemplateRegionKind Kind { get; init; } + public required IReadOnlyList Pages { get; init; } + public required string Fingerprint { get; init; } + public string? Story { get; init; } + public string? Location { get; init; } + public double Confidence { get; init; } +} + +public sealed record RepeatingRegion +{ + public required string Id { get; init; } + public required string Name { get; init; } + public required IReadOnlyList Pages { get; init; } + public required string Fingerprint { get; init; } + public required TemplateRegionKind Kind { get; init; } + public string? Location { get; init; } +} + +public sealed record PageArchetype +{ + public required string Id { get; init; } + public required string Name { get; init; } + public bool IsFirstPage { get; init; } + public bool IsRepeating { get; init; } + public required IReadOnlyList ExamplePages { get; init; } + public required IReadOnlyList FixedRegionIds { get; init; } + public required IReadOnlyList SlotIds { get; init; } +} + +public sealed record TemplateFieldBinding +{ + public required string SourceField { get; init; } + public required string TargetSlot { get; init; } + public required string Scope { get; init; } + public string? SourceStyle { get; init; } + public string? TargetStyle { get; init; } +} + +public sealed record SourceDocumentStructure +{ + public required string SourcePath { get; init; } + public required string Sha256 { get; init; } + public required string NormalizedPath { get; init; } + public required string NormalizedSha256 { get; init; } + public required int PageCount { get; init; } + public required int SectionCount { get; init; } + public required IReadOnlyList Stories { get; init; } + public required IReadOnlyList RepeatedFingerprints { get; init; } + public required DocumentInspectionResult Inspection { get; init; } +} + +public sealed record TemplateLearningSuggestion +{ + public required string SuggestionType { get; init; } + public required string SourceLocation { get; init; } + public string? TargetSlot { get; init; } + public string? ProposedName { get; init; } + public double Confidence { get; init; } + public required string Reason { get; init; } + public bool RequiresConfirmation { get; init; } = true; +} + +public sealed record LearnedTemplate +{ + public required string TemplateId { get; init; } + public required string Name { get; init; } + public required string SourceFileName { get; init; } + public required string SourceSha256 { get; init; } + public required string NormalizedSha256 { get; init; } + public required string CreatedAt { get; init; } + public string MinimumEngineVersion { get; init; } = "1.0"; + public required SourceDocumentStructure Source { get; init; } + public required IReadOnlyList PageArchetypes { get; init; } + public required IReadOnlyList FixedRegions { get; init; } + public required IReadOnlyList RepeatingRegions { get; init; } + public required IReadOnlyList Slots { get; init; } + public required IReadOnlyList FieldBindings { get; init; } + public required IReadOnlyList Suggestions { get; init; } +} + +public sealed record DocumentNormalizationRequest +{ + public required string SourcePath { get; init; } + public required string WorkingDirectory { get; init; } + public bool OpenReadOnly { get; init; } = true; + public bool DisableMacros { get; init; } = true; +} + +public sealed record DocumentNormalizationResult +{ + public required string SourcePath { get; init; } + public required string NormalizedPath { get; init; } + public required string SourceSha256 { get; init; } + public required string NormalizedSha256 { get; init; } + public required bool WasConverted { get; init; } + public required TimeSpan Duration { get; init; } + public int? PageCount { get; init; } +} + +public sealed record TemplateLearningRequest +{ + public required string SourcePath { get; init; } + public required string TemplateName { get; init; } + public required string OutputPackagePath { get; init; } + public bool CreateDefaultSlots { get; init; } = true; +} + +public sealed record TemplateGenerationRequest +{ + public required string TemplatePackagePath { get; init; } + public required string ContentSourcePath { get; init; } + public required string OutputDocxPath { get; init; } + public IReadOnlyDictionary ScalarValues { get; init; } = new Dictionary(StringComparer.OrdinalIgnoreCase); +} + +public sealed record TemplateGenerationResult +{ + public required string OutputDocxPath { get; init; } + public required string OutputPdfPath { get; init; } + public required string GenerationLogPath { get; init; } + public required string ValidationLogPath { get; init; } + public required IReadOnlyList Validation { get; init; } +} diff --git a/DocumentAutomation/src/DocxWorkbench.App/MainForm.cs b/DocumentAutomation/src/DocxWorkbench.App/MainForm.cs index 1a4d9ae..6e1945b 100644 --- a/DocumentAutomation/src/DocxWorkbench.App/MainForm.cs +++ b/DocumentAutomation/src/DocxWorkbench.App/MainForm.cs @@ -9,47 +9,63 @@ namespace DocxWorkbench.App; public sealed class MainForm : Form { - private readonly TextBox _template = new() { Width = 520 }; private readonly TextBox _body = new() { Width = 520 }; private readonly TextBox _output = new() { Width = 520 }; private readonly RichTextBox _structure = new() { Dock = DockStyle.Fill, ReadOnly = true }; private readonly RichTextBox _bodyPreview = new() { Dock = DockStyle.Fill, ReadOnly = true }; private readonly TextBox _log = new() { Multiline = true, ScrollBars = ScrollBars.Vertical, Dock = DockStyle.Fill, ReadOnly = true }; private readonly TextBox _technicalError = new() { Multiline = true, ScrollBars = ScrollBars.Vertical, Dock = DockStyle.Fill, ReadOnly = true }; - private readonly Label _status = new() { AutoSize = true, Text = "Ready" }; private readonly Button _analyze; private readonly Button _build; private readonly Button _validate; private readonly Button _openDocx; private readonly Button _openPdf; private readonly WorkbenchStateController _state = new(); private readonly IDocumentInspector _docx = new OpenXmlDocumentInspector(); private readonly IDocumentInspector _txt = new TextDocumentInspector(); private readonly IWordDocumentService _word = new WordDocumentService(); private DocumentInspectionResult? _templateResult; private DocumentInspectionResult? _bodyResult; private string? _builtDocx; private string? _builtPdf; + private readonly ComboBox _mode = new() { DropDownStyle = ComboBoxStyle.DropDownList, Width = 230 }; + private readonly TextBox _template = new() { Width = 560 }; + private readonly TextBox _body = new() { Width = 560 }; + private readonly TextBox _package = new() { Width = 560 }; + private readonly TextBox _templateName = new() { Width = 560, Text = "Learned document template" }; + private readonly TextBox _output = new() { Width = 560 }; + private readonly RichTextBox _structure = new() { Dock = DockStyle.Fill, ReadOnly = true }; + private readonly RichTextBox _preview = new() { Dock = DockStyle.Fill, ReadOnly = true }; + private readonly TextBox _log = new() { Multiline = true, ScrollBars = ScrollBars.Vertical, Dock = DockStyle.Fill, ReadOnly = true }; + private readonly TextBox _technicalError = new() { Multiline = true, ScrollBars = ScrollBars.Vertical, Dock = DockStyle.Fill, ReadOnly = true }; + private readonly Label _status = new() { AutoSize = true, Text = "Ready" }; + private readonly Button _analyze; private readonly Button _learn; private readonly Button _generate; private readonly Button _build; private readonly Button _validate; private readonly Button _openDocx; private readonly Button _openPdf; + private readonly WorkbenchStateController _state = new(); + private readonly IDocumentInspector _docx = new OpenXmlDocumentInspector(); private readonly IDocumentInspector _txt = new TextDocumentInspector(); private readonly IWordDocumentService _word = new WordDocumentService(); + private DocumentInspectionResult? _templateResult; private DocumentInspectionResult? _bodyResult; private string? _builtDocx; private string? _builtPdf; public MainForm() { - Text = "DOCX Workbench"; Width = 1100; Height = 720; StartPosition = FormStartPosition.CenterScreen; var grid = new TableLayoutPanel { Dock = DockStyle.Top, Height = 145, ColumnCount = 3, RowCount = 4, Padding = new Padding(8) }; grid.ColumnStyles.Add(new ColumnStyle(SizeType.Absolute, 150)); grid.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 100)); grid.ColumnStyles.Add(new ColumnStyle(SizeType.Absolute, 110)); AddRow(grid, 0, "Template DOCX", _template, BrowseTemplate); AddRow(grid, 1, "Document body", _body, BrowseBody); AddRow(grid, 2, "Output DOCX", _output, BrowseOutput); - var buttons = new FlowLayoutPanel { Dock = DockStyle.Fill }; _analyze = Button("Analyze", async (_, _) => await AnalyzeAsync()); _build = Button("Build", async (_, _) => await BuildAsync()); _validate = Button("Validate", async (_, _) => await ValidateAsync()); _openDocx = Button("Open DOCX", (_, _) => Open(_builtDocx)); _openPdf = Button("Open PDF", (_, _) => Open(_builtPdf)); buttons.Controls.AddRange([_analyze, _build, _validate, _openDocx, _openPdf]); grid.Controls.Add(buttons, 1, 3); grid.SetColumnSpan(buttons, 2); - var tabs = new TabControl { Dock = DockStyle.Fill }; AddTab(tabs, "Template structure", _structure); AddTab(tabs, "Body preview", _bodyPreview); AddTab(tabs, "Result", new Label { Text = "The result becomes available after a successful build.", AutoSize = true }); AddTab(tabs, "Violations", new Label { Text = "No validation has been run.", AutoSize = true }); AddTab(tabs, "Technical error", _technicalError); AddTab(tabs, "Log", _log); Controls.Add(tabs); Controls.Add(grid); Controls.Add(_status); _status.Dock = DockStyle.Bottom; - foreach (var box in new[] { _template, _body, _output }) box.TextChanged += (_, _) => InputsChanged(); RefreshButtons(); + Text = "DOCX Workbench"; Width = 1200; Height = 780; StartPosition = FormStartPosition.CenterScreen; + _mode.Items.AddRange(["Template learning", "Document generation", "Quick DOC_BODY"]); _mode.SelectedIndex = 2; _mode.SelectedIndexChanged += (_, _) => RefreshButtons(); + var grid = new TableLayoutPanel { Dock = DockStyle.Top, Height = 245, ColumnCount = 3, RowCount = 7, Padding = new Padding(8) }; grid.ColumnStyles.Add(new ColumnStyle(SizeType.Absolute, 155)); grid.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 100)); grid.ColumnStyles.Add(new ColumnStyle(SizeType.Absolute, 110)); + AddControlRow(grid, 0, "Mode", _mode, null); AddControlRow(grid, 1, "Template DOC/DOCX", _template, BrowseTemplate); AddControlRow(grid, 2, "Template name", _templateName, null); AddControlRow(grid, 3, "Content DOC/DOCX/TXT", _body, BrowseBody); AddControlRow(grid, 4, "Package .dtemplate", _package, BrowsePackage); AddControlRow(grid, 5, "Output DOCX", _output, BrowseOutput); + var buttons = new FlowLayoutPanel { Dock = DockStyle.Fill }; _analyze = Button("Analyze", async (_, _) => await AnalyzeAsync()); _learn = Button("Learn template", async (_, _) => await LearnAsync()); _generate = Button("Generate", async (_, _) => await GenerateAsync()); _build = Button("Quick build", async (_, _) => await QuickBuildAsync()); _validate = Button("Validate", async (_, _) => await ValidateAsync()); _openDocx = Button("Open DOCX", (_, _) => Open(_builtDocx)); _openPdf = Button("Open PDF", (_, _) => Open(_builtPdf)); buttons.Controls.AddRange([_analyze, _learn, _generate, _build, _validate, _openDocx, _openPdf]); grid.Controls.Add(buttons, 1, 6); grid.SetColumnSpan(buttons, 2); + var tabs = new TabControl { Dock = DockStyle.Fill }; AddTab(tabs, "Structure", _structure); AddTab(tabs, "Preview", _preview); AddTab(tabs, "Log", _log); AddTab(tabs, "Technical error", _technicalError); Controls.Add(tabs); Controls.Add(grid); _status.Dock = DockStyle.Bottom; Controls.Add(_status); foreach (var box in new[] { _template, _body, _package, _output }) box.TextChanged += (_, _) => InputsChanged(); FormClosed += (_, _) => (_word as IDisposable)?.Dispose(); RefreshButtons(); } private static void AddTab(TabControl tabs, string title, Control content) { var tab = new TabPage(title); tab.Controls.Add(content); tabs.TabPages.Add(tab); } - private static void AddRow(TableLayoutPanel panel, int row, string title, TextBox box, EventHandler browse) { panel.Controls.Add(new Label { Text = title, AutoSize = true, Anchor = AnchorStyles.Left }, 0, row); panel.Controls.Add(box, 1, row); var button = new Button { Text = "Browse...", Width = 95 }; button.Click += browse; panel.Controls.Add(button, 2, row); } + private static void AddControlRow(TableLayoutPanel panel, int row, string title, Control control, EventHandler? browse) { panel.Controls.Add(new Label { Text = title, AutoSize = true, Anchor = AnchorStyles.Left }, 0, row); panel.Controls.Add(control, 1, row); if (browse is not null) { var button = new Button { Text = "Browse...", Width = 95 }; button.Click += browse; panel.Controls.Add(button, 2, row); } } private static Button Button(string text, EventHandler handler) { var button = new Button { Text = text, AutoSize = true }; button.Click += handler; return button; } - private void BrowseTemplate(object? sender, EventArgs e) { using var dialog = new OpenFileDialog { Filter = "DOCX|*.docx" }; if (dialog.ShowDialog() == DialogResult.OK) _template.Text = dialog.FileName; } - private void BrowseBody(object? sender, EventArgs e) { using var dialog = new OpenFileDialog { Filter = "DOCX or TXT|*.docx;*.txt" }; if (dialog.ShowDialog() == DialogResult.OK) _body.Text = dialog.FileName; } + private void BrowseTemplate(object? sender, EventArgs e) { using var dialog = new OpenFileDialog { Filter = "Word document|*.doc;*.docx" }; if (dialog.ShowDialog() == DialogResult.OK) _template.Text = dialog.FileName; } + private void BrowseBody(object? sender, EventArgs e) { using var dialog = new OpenFileDialog { Filter = "Word or text|*.doc;*.docx;*.txt" }; if (dialog.ShowDialog() == DialogResult.OK) _body.Text = dialog.FileName; } + private void BrowsePackage(object? sender, EventArgs e) { using var dialog = new OpenFileDialog { Filter = "Learned template|*.dtemplate" }; if (dialog.ShowDialog() == DialogResult.OK) _package.Text = dialog.FileName; } private void BrowseOutput(object? sender, EventArgs e) { using var dialog = new SaveFileDialog { Filter = "DOCX|*.docx", DefaultExt = "docx" }; if (dialog.ShowDialog() == DialogResult.OK) _output.Text = dialog.FileName; } + private async Task LearnAsync() + { + try { _status.Text = "Learning template..."; using var normalizer = new WordDocumentNormalizer(); var service = new TemplateLearningService(normalizer, new TemplatePackageStore(), _docx); var learned = await service.LearnAsync(new TemplateLearningRequest { SourcePath = _template.Text, TemplateName = _templateName.Text, OutputPackagePath = _package.Text }, null, CancellationToken.None); _structure.Text = JsonSerializer.Serialize(learned, new JsonSerializerOptions { WriteIndented = true }); AppendLog($"Template package created: {_package.Text}"); _status.Text = "Template learned"; } catch (Exception ex) { ShowError(ex, "Template learning failed"); } + } + + private async Task GenerateAsync() + { + try { _status.Text = "Generating..."; using var normalizer = new WordDocumentNormalizer(); using var generator = new TemplateDocumentGenerator(new TemplatePackageStore(), normalizer, _docx); var result = await generator.GenerateAsync(new TemplateGenerationRequest { TemplatePackagePath = _package.Text, ContentSourcePath = _body.Text, OutputDocxPath = _output.Text }, null, CancellationToken.None); _builtDocx = result.OutputDocxPath; _builtPdf = result.OutputPdfPath; _preview.Text = JsonSerializer.Serialize(result.Validation, new JsonSerializerOptions { WriteIndented = true }); AppendLog(JsonSerializer.Serialize(result, new JsonSerializerOptions { WriteIndented = true })); _status.Text = "Generation succeeded"; } catch (Exception ex) { ShowError(ex, "Generation failed"); } + } + private async Task AnalyzeAsync() { - _templateResult = null; _bodyResult = null; _builtDocx = null; _builtPdf = null; _state.InputsSelected(File.Exists(_template.Text) && File.Exists(_body.Text)); RefreshButtons(); _status.Text = "Analyzing..."; - var template = await InspectPathAsync(_template.Text); var body = await InspectPathAsync(_body.Text); if (template.Error is null) { _templateResult = template.Result; ShowStructure(template.Result); } if (body.Error is null) { _bodyResult = body.Result; ShowBody(body.Result, _body.Text); } LogInspection("Template", _template.Text, template); LogInspection("Body", _body.Text, body); var insertion = _templateResult?.InsertionPoint.Type != "None"; _state.AnalysisCompleted(_templateResult is not null, _bodyResult is not null, insertion); RefreshButtons(); _status.Text = _state.State == WorkbenchState.ReadyToBuild ? "Ready to build" : "Analysis completed with errors"; if (!insertion && _templateResult is not null) AppendLog("No DOC_BODY insertion point found. Word was not started."); + _status.Text = "Analyzing..."; var template = await InspectPathAsync(_template.Text); var body = await InspectPathAsync(_body.Text); _templateResult = template.Result; _bodyResult = body.Result; if (_templateResult is not null) ShowStructure(_templateResult); if (_bodyResult is not null) _preview.Text = _body.Text.EndsWith(".txt", StringComparison.OrdinalIgnoreCase) ? File.ReadAllText(_body.Text) : JsonSerializer.Serialize(_bodyResult, new JsonSerializerOptions { WriteIndented = true }); LogInspection("Template", template); LogInspection("Body", body); _state.InputsSelected(template.Result is not null && body.Result is not null); _state.AnalysisCompleted(template.Result is not null, body.Result is not null, template.Result?.InsertionPoint.Type != "None"); _status.Text = _state.State == WorkbenchState.ReadyToBuild ? "Ready" : "Analysis completed with warnings"; RefreshButtons(); } - private Task<(DocumentInspectionResult? Result, Exception? Error, string Inspector)> InspectPathAsync(string path) => Task.Run(() => { try { if (path.EndsWith(".docx", StringComparison.OrdinalIgnoreCase)) return (_docx.Inspect(path), (Exception?)null, "OpenXmlDocumentInspector"); if (path.EndsWith(".txt", StringComparison.OrdinalIgnoreCase)) return (_txt.Inspect(path), (Exception?)null, "TextDocumentInspector"); throw new UnsupportedDocumentFormatException($"Unsupported format {Path.GetExtension(path)}. Choose DOCX or TXT."); } catch (Exception ex) { return ((DocumentInspectionResult?)null, ex, path.EndsWith(".txt", StringComparison.OrdinalIgnoreCase) ? "TextDocumentInspector" : "OpenXmlDocumentInspector"); } }); - private void ShowStructure(DocumentInspectionResult? result) { if (result is null) return; _structure.Text = JsonSerializer.Serialize(new { result.FilePath, result.Sections, result.Paragraphs, result.Tables, result.Headers, result.Footers, result.Fields, result.ImageSummary, result.SectionAssignments, result.InsertionPoint, result.StyleDetails, result.Warnings }, new JsonSerializerOptions { WriteIndented = true }); } - private void ShowBody(DocumentInspectionResult? result, string path) { if (path.EndsWith(".txt", StringComparison.OrdinalIgnoreCase)) _bodyPreview.Text = File.ReadAllText(path); else if (result is not null) _bodyPreview.Text = JsonSerializer.Serialize(result, new JsonSerializerOptions { WriteIndented = true }); } - private void LogInspection(string label, string path, (DocumentInspectionResult? Result, Exception? Error, string Inspector) value) { AppendLog(JsonSerializer.Serialize(new { label, path, inspector = value.Inspector, result = value.Result, error = value.Error?.Message }, new JsonSerializerOptions { WriteIndented = true })); if (value.Error is not null) _technicalError.Text = value.Error.ToString(); } - - private async Task BuildAsync() - { - if (!_state.CanBuild) return; _state.BeginBuild(); RefreshButtons(); _status.Text = "Building..."; try { var result = await _word.BuildDocumentAsync(new WordDocumentBuildRequest { TemplatePath = _template.Text, BodyPath = _body.Text, OutputDocxPath = _output.Text }, new Progress(x => _status.Text = x), CancellationToken.None); var pdf = await _word.ExportPdfAsync(result.OutputDocxPath, Path.ChangeExtension(result.OutputDocxPath, ".pdf"), CancellationToken.None); _builtDocx = result.OutputDocxPath; _builtPdf = pdf.PdfPath; _state.BuildSucceeded(); AppendLog(JsonSerializer.Serialize(new { result, pdf }, new JsonSerializerOptions { WriteIndented = true })); _status.Text = "Build succeeded"; } catch (Exception ex) { _state.BuildFailed(); _technicalError.Text = ex.ToString(); AppendLog("Build failed: " + ex.Message); _status.Text = "Build failed"; MessageBox.Show(ex.Message, "Build failed", MessageBoxButtons.OK, MessageBoxIcon.Error); } finally { RefreshButtons(); } - } - - private async Task ValidateAsync() - { - if (!_state.CanValidate || _builtDocx is null || !File.Exists(_builtDocx)) return; _state.BeginValidation(); RefreshButtons(); _status.Text = "Validating..."; try { var result = await Task.Run(() => _docx.Inspect(_builtDocx)); _state.ValidationCompleted(true); AppendLog(JsonSerializer.Serialize(new { label = "Output", result }, new JsonSerializerOptions { WriteIndented = true })); _status.Text = "Validation succeeded"; } catch (Exception ex) { _state.ValidationCompleted(false); _technicalError.Text = ex.ToString(); AppendLog("Validation failed: " + ex.Message); _status.Text = "Validation failed"; MessageBox.Show(ex.Message, "Validation failed", MessageBoxButtons.OK, MessageBoxIcon.Error); } finally { RefreshButtons(); } - } - - private void InputsChanged() { _templateResult = null; _bodyResult = null; _builtDocx = null; _builtPdf = null; _state.InputsSelected(File.Exists(_template.Text) && File.Exists(_body.Text)); RefreshButtons(); } - private void RefreshButtons() { _analyze.Enabled = File.Exists(_template.Text) && File.Exists(_body.Text); _build.Enabled = _state.CanBuild; _validate.Enabled = _state.CanValidate && _builtDocx is not null && File.Exists(_builtDocx); _openDocx.Enabled = _state.CanOpenDocx && _builtDocx is not null && File.Exists(_builtDocx); _openPdf.Enabled = _state.CanOpenPdf && _builtPdf is not null && File.Exists(_builtPdf); } + private Task<(DocumentInspectionResult? Result, Exception? Error, string Inspector)> InspectPathAsync(string path) => Task.Run(() => { try { if (path.EndsWith(".docx", StringComparison.OrdinalIgnoreCase)) return (_docx.Inspect(path), (Exception?)null, "OpenXmlDocumentInspector"); if (path.EndsWith(".txt", StringComparison.OrdinalIgnoreCase)) return (_txt.Inspect(path), (Exception?)null, "TextDocumentInspector"); throw new UnsupportedDocumentFormatException($"Unsupported format {Path.GetExtension(path)}. Use DOC/DOCX/TXT or normalize DOC first."); } catch (Exception ex) { return ((DocumentInspectionResult?)null, ex, "format-router"); } }); + private async Task QuickBuildAsync() { if (!_state.CanBuild) { await AnalyzeAsync(); if (!_state.CanBuild) return; } _state.BeginBuild(); RefreshButtons(); try { var result = await _word.BuildDocumentAsync(new WordDocumentBuildRequest { TemplatePath = _template.Text, BodyPath = _body.Text, OutputDocxPath = _output.Text }, null, CancellationToken.None); var pdf = await _word.ExportPdfAsync(result.OutputDocxPath, Path.ChangeExtension(result.OutputDocxPath, ".pdf"), CancellationToken.None); _builtDocx = result.OutputDocxPath; _builtPdf = pdf.PdfPath; _state.BuildSucceeded(); AppendLog(JsonSerializer.Serialize(new { result, pdf }, new JsonSerializerOptions { WriteIndented = true })); _status.Text = "Quick build succeeded"; } catch (Exception ex) { _state.BuildFailed(); ShowError(ex, "Build failed"); } finally { RefreshButtons(); } } + private async Task ValidateAsync() { if (!_state.CanValidate || _builtDocx is null || !File.Exists(_builtDocx)) return; try { var result = await Task.Run(() => _docx.Inspect(_builtDocx)); AppendLog(JsonSerializer.Serialize(new { label = "Output", result }, new JsonSerializerOptions { WriteIndented = true })); _status.Text = "Validation succeeded"; } catch (Exception ex) { ShowError(ex, "Validation failed"); } } + private void ShowStructure(DocumentInspectionResult result) => _structure.Text = JsonSerializer.Serialize(new { result.FilePath, result.Sections, result.Paragraphs, result.Tables, result.Headers, result.Footers, result.Fields, result.ImageSummary, result.SectionAssignments, result.StyleDetails, result.Warnings }, new JsonSerializerOptions { WriteIndented = true }); + private void LogInspection(string label, (DocumentInspectionResult? Result, Exception? Error, string Inspector) value) { AppendLog(JsonSerializer.Serialize(new { label, inspector = value.Inspector, result = value.Result, error = value.Error?.Message }, new JsonSerializerOptions { WriteIndented = true })); if (value.Error is not null) _technicalError.Text = value.Error.Message; } + private void InputsChanged() { _builtDocx = null; _builtPdf = null; RefreshButtons(); } + private void RefreshButtons() { var mode = _mode.SelectedIndex; var templateSupported = _template.Text.EndsWith(".doc", StringComparison.OrdinalIgnoreCase) || _template.Text.EndsWith(".docx", StringComparison.OrdinalIgnoreCase); _analyze.Enabled = mode != 1 && File.Exists(_template.Text) && File.Exists(_body.Text); _learn.Enabled = mode == 0 && templateSupported && File.Exists(_template.Text) && !string.IsNullOrWhiteSpace(_package.Text); _generate.Enabled = mode == 1 && File.Exists(_package.Text) && File.Exists(_body.Text) && !string.IsNullOrWhiteSpace(_output.Text); _build.Enabled = mode == 2 && _state.CanBuild; _validate.Enabled = _state.CanValidate && _builtDocx is not null && File.Exists(_builtDocx); _openDocx.Enabled = _builtDocx is not null && File.Exists(_builtDocx); _openPdf.Enabled = _builtPdf is not null && File.Exists(_builtPdf); } + private void ShowError(Exception ex, string title) { _technicalError.Text = ex.ToString(); AppendLog(title + ": " + ex.Message); _status.Text = title; MessageBox.Show(ex.Message, title, MessageBoxButtons.OK, MessageBoxIcon.Error); } private void AppendLog(string text) => _log.AppendText(text + Environment.NewLine); private static void Open(string? path) { if (!string.IsNullOrWhiteSpace(path) && File.Exists(path)) Process.Start(new ProcessStartInfo(path) { UseShellExecute = true }); } } diff --git a/DocumentAutomation/src/DocxWorkbench.Core/Contracts.cs b/DocumentAutomation/src/DocxWorkbench.Core/Contracts.cs index eb42bf8..938e620 100644 --- a/DocumentAutomation/src/DocxWorkbench.Core/Contracts.cs +++ b/DocumentAutomation/src/DocxWorkbench.Core/Contracts.cs @@ -39,6 +39,27 @@ public interface IDocxBodyImporter Task ImportAsync(DocxBodyImportRequest request, IProgress? progress, CancellationToken cancellationToken); } +public interface IWordDocumentNormalizer +{ + Task NormalizeAsync(DocumentNormalizationRequest request, IProgress? progress, CancellationToken cancellationToken); +} + +public interface ITemplateLearningService +{ + Task LearnAsync(TemplateLearningRequest request, IProgress? progress, CancellationToken cancellationToken); +} + +public interface ITemplatePackageStore +{ + void Save(string packagePath, LearnedTemplate template, string normalizedDocxPath); + (LearnedTemplate Template, string TemplateDocxPath, string WorkingDirectory) Open(string packagePath); +} + +public interface ITemplateDocumentGenerator +{ + Task GenerateAsync(TemplateGenerationRequest request, IProgress? progress, CancellationToken cancellationToken); +} + public static class DocumentInputValidation { public static void Validate(WordDocumentBuildRequest request) diff --git a/DocumentAutomation/src/DocxWorkbench.OpenXml/TemplateLearningService.cs b/DocumentAutomation/src/DocxWorkbench.OpenXml/TemplateLearningService.cs new file mode 100644 index 0000000..fe4b538 --- /dev/null +++ b/DocumentAutomation/src/DocxWorkbench.OpenXml/TemplateLearningService.cs @@ -0,0 +1,73 @@ +using System.IO.Compression; +using System.Security.Cryptography; +using System.Text.Json; +using DocumentAutomation.Shared; +using DocumentFormat.OpenXml.Packaging; +using DocumentFormat.OpenXml.Wordprocessing; +using DocxWorkbench.Core; + +namespace DocxWorkbench.OpenXml; + +public sealed class TemplatePackageStore : ITemplatePackageStore +{ + private static readonly JsonSerializerOptions JsonOptions = new() { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }; + + public void Save(string packagePath, LearnedTemplate template, string normalizedDocxPath) + { + Directory.CreateDirectory(Path.GetDirectoryName(Path.GetFullPath(packagePath))!); + if (File.Exists(packagePath)) File.Delete(packagePath); + using var archive = ZipFile.Open(packagePath, ZipArchiveMode.Create); + WriteJson(archive, "manifest.json", new { schemaVersion = "1.0", templateId = template.TemplateId, name = template.Name, sourceFileName = template.SourceFileName, sourceSha256 = template.SourceSha256, normalizedSha256 = template.NormalizedSha256, createdAt = template.CreatedAt, minimumEngineVersion = template.MinimumEngineVersion }); + WriteJson(archive, "source-metadata.json", template.Source); + WriteJson(archive, "page-archetypes.json", template.PageArchetypes); + WriteJson(archive, "fixed-regions.json", template.FixedRegions.Concat(template.RepeatingRegions.Select(x => new FixedRegion { Id = x.Id, Name = x.Name, Kind = x.Kind, Pages = x.Pages, Fingerprint = x.Fingerprint, Location = x.Location })).ToArray()); + WriteJson(archive, "slots.json", template.Slots); + WriteJson(archive, "field-bindings.json", template.FieldBindings); + WriteJson(archive, "mapping-rules.json", template.Suggestions); + WriteJson(archive, "validation-profile.json", new { schemaVersion = "1.0", requiredSlots = template.Slots.Where(x => x.IsRequired).Select(x => x.ContentControlTag).ToArray(), allowTrackedChanges = false, allowExternalLinks = false }); + var entry = archive.CreateEntry("template.docx", CompressionLevel.Optimal); using var input = File.OpenRead(normalizedDocxPath); using var output = entry.Open(); input.CopyTo(output); + } + + public (LearnedTemplate Template, string TemplateDocxPath, string WorkingDirectory) Open(string packagePath) + { + if (!File.Exists(packagePath)) throw new FileNotFoundException("Template package was not found.", packagePath); + var work = Path.Combine(Path.GetTempPath(), "DocumentAutomation", "template-" + Guid.NewGuid().ToString("N")); Directory.CreateDirectory(work); ZipFile.ExtractToDirectory(packagePath, work); + var manifest = ReadJson(Path.Combine(work, "manifest.json")); if (manifest.SchemaVersion != "1.0") throw new InvalidDataException($"Unsupported template schema: {manifest.SchemaVersion}"); + var template = new LearnedTemplate { TemplateId = manifest.TemplateId, Name = manifest.Name, SourceFileName = manifest.SourceFileName, SourceSha256 = manifest.SourceSha256, NormalizedSha256 = manifest.NormalizedSha256, CreatedAt = manifest.CreatedAt, MinimumEngineVersion = manifest.MinimumEngineVersion, Source = ReadJson(Path.Combine(work, "source-metadata.json")), PageArchetypes = ReadJson>(Path.Combine(work, "page-archetypes.json")), FixedRegions = ReadJson>(Path.Combine(work, "fixed-regions.json")).Where(x => x.Kind != TemplateRegionKind.Repeating).ToArray(), RepeatingRegions = ReadJson>(Path.Combine(work, "fixed-regions.json")).Where(x => x.Kind == TemplateRegionKind.Repeating).Select(x => new RepeatingRegion { Id = x.Id, Name = x.Name, Pages = x.Pages, Fingerprint = x.Fingerprint, Kind = x.Kind, Location = x.Location }).ToArray(), Slots = ReadJson>(Path.Combine(work, "slots.json")), FieldBindings = ReadJson>(Path.Combine(work, "field-bindings.json")), Suggestions = ReadJson>(Path.Combine(work, "mapping-rules.json")) }; + var docx = Path.Combine(work, "template.docx"); if (!File.Exists(docx)) throw new InvalidDataException("Template package does not contain template.docx."); return (template, docx, work); + } + + private static void WriteJson(ZipArchive archive, string name, T value) { var entry = archive.CreateEntry(name, CompressionLevel.Optimal); using var writer = new StreamWriter(entry.Open()); writer.Write(JsonSerializer.Serialize(value, JsonOptions)); } + private static T ReadJson(string path) => JsonSerializer.Deserialize(File.ReadAllText(path), JsonOptions) ?? throw new InvalidDataException($"Invalid template package entry: {Path.GetFileName(path)}"); + private sealed record Manifest(string SchemaVersion, string TemplateId, string Name, string SourceFileName, string SourceSha256, string NormalizedSha256, string CreatedAt, string MinimumEngineVersion); +} + +public sealed class TemplateLearningService : ITemplateLearningService +{ + private readonly IWordDocumentNormalizer _normalizer; + private readonly ITemplatePackageStore _packages; + private readonly IDocumentInspector _inspector; + public TemplateLearningService(IWordDocumentNormalizer normalizer, ITemplatePackageStore packages, IDocumentInspector? inspector = null) { _normalizer = normalizer; _packages = packages; _inspector = inspector ?? new OpenXmlDocumentInspector(); } + + public async Task LearnAsync(TemplateLearningRequest request, IProgress? progress, CancellationToken cancellationToken) + { + if (!File.Exists(request.SourcePath)) throw new FileNotFoundException("Template source was not found.", request.SourcePath); + var normalized = await _normalizer.NormalizeAsync(new DocumentNormalizationRequest { SourcePath = request.SourcePath, WorkingDirectory = Path.Combine(Path.GetTempPath(), "DocumentAutomation", "learning-" + Guid.NewGuid().ToString("N")) }, progress, cancellationToken); + try + { + progress?.Report(new WordOperationProgress { Stage = "AnalyzeTemplate", Message = "Inspecting normalized template" }); + EnsureBodyMarker(normalized.NormalizedPath); var normalizedHash = Convert.ToHexString(SHA256.HashData(File.ReadAllBytes(normalized.NormalizedPath))); var inspection = _inspector.Inspect(normalized.NormalizedPath) with { FilePath = "template.docx" }; var pages = Math.Max(1, normalized.PageCount ?? inspection.Sections); var archetypes = CreateArchetypes(pages, inspection); var repeated = FindRepeatedRegions(inspection); var slots = request.CreateDefaultSlots ? DefaultSlots() : []; + var source = new SourceDocumentStructure { SourcePath = Path.GetFileName(request.SourcePath), Sha256 = normalized.SourceSha256, NormalizedPath = "template.docx", NormalizedSha256 = normalizedHash, PageCount = pages, SectionCount = inspection.Sections, Stories = ["wdMainTextStory", "wdPrimaryHeaderStory", "wdFirstPageHeaderStory", "wdEvenPagesHeaderStory", "wdPrimaryFooterStory", "wdFootnotesStory", "wdEndnotesStory", "wdCommentsStory", "wdTextFrameStory"], RepeatedFingerprints = repeated.Select(x => x.Fingerprint).ToArray(), Inspection = inspection }; + var learned = new LearnedTemplate { TemplateId = Guid.NewGuid().ToString("D"), Name = request.TemplateName, SourceFileName = Path.GetFileName(request.SourcePath), SourceSha256 = normalized.SourceSha256, NormalizedSha256 = normalizedHash, CreatedAt = DateTimeOffset.Now.ToString("O"), Source = source, PageArchetypes = archetypes, FixedRegions = [], RepeatingRegions = repeated, Slots = slots, FieldBindings = [], Suggestions = BuildSuggestions(inspection, slots) }; + _packages.Save(request.OutputPackagePath, learned, normalized.NormalizedPath); return learned; + } + finally { TryDeleteDirectory(normalized.NormalizedPath, normalized.SourcePath); } + } + + private static void EnsureBodyMarker(string path) { using var doc = WordprocessingDocument.Open(path, true); var main = doc.MainDocumentPart ?? throw new InvalidDataException("Main document part is missing."); var body = main.Document.Body ?? throw new InvalidDataException("Document body is missing."); if (main.Document.InnerText.Contains("{{DOC_BODY}}", StringComparison.Ordinal)) return; body.AppendChild(new Paragraph(new Run(new Text("{{DOC_BODY}}")))); main.Document.Save(); } + private static IReadOnlyList CreateArchetypes(int pages, DocumentInspectionResult inspection) { var result = new List { new() { Id = "FirstPage", Name = "FirstPage", IsFirstPage = true, IsRepeating = false, ExamplePages = [1], FixedRegionIds = [], SlotIds = [] } }; if (pages > 1) result.Add(new PageArchetype { Id = "RepeatingPage", Name = "RepeatingPage", IsRepeating = true, ExamplePages = Enumerable.Range(2, pages - 1).ToArray(), FixedRegionIds = [], SlotIds = [] }); return result; } + private static IReadOnlyList FindRepeatedRegions(DocumentInspectionResult inspection) => inspection.SectionAssignments.GroupBy(x => string.Join("|", x.Headers.Concat(x.Footers))).Where(x => x.Count() > 1 && x.Key.Length > 0).Select((x, i) => new RepeatingRegion { Id = "region-" + (i + 1), Name = "Repeated header/footer", Pages = x.Select(y => y.Section).ToArray(), Fingerprint = Convert.ToHexString(SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(x.Key))), Kind = TemplateRegionKind.Repeating, Location = "section assignments" }).ToArray(); + private static IReadOnlyList DefaultSlots() => new[] { "PROJECT.CODE", "PROJECT.NAME", "PROJECT.OBJECT", "DOCUMENT.CODE", "DOCUMENT.TITLE", "DOCUMENT.STAGE", "DOCUMENT.SHEET", "DOCUMENT.SHEET_COUNT", "REGISTER.CONTENTS", "REGISTER.MAIN_DRAWING_SETS", "REGISTER.DRAWINGS", "REGISTER.REFERENCE_DOCUMENTS", "DOCUMENT.GENERAL_NOTES", "DOCUMENT.MAIN_BODY" }.Select(x => new TemplateSlot { Id = x, Name = x, Kind = x == "DOCUMENT.MAIN_BODY" ? TemplateSlotKind.MainBody : TemplateSlotKind.PlainText, ContentControlTag = x, IsRequired = x is "DOCUMENT.CODE" or "DOCUMENT.TITLE" or "DOCUMENT.MAIN_BODY", Occurrences = [] }).ToArray(); + private static IReadOnlyList BuildSuggestions(DocumentInspectionResult inspection, IReadOnlyList slots) => [new TemplateLearningSuggestion { SuggestionType = "PageArchetype", SourceLocation = "document", ProposedName = inspection.Sections > 1 ? "FirstPage + RepeatingPage" : "SinglePage", Confidence = 0.5, Reason = "Initial deterministic proposal based on document sections." }]; + private static void TryDeleteDirectory(string normalized, string source) { var directory = Path.GetDirectoryName(normalized); if (directory is not null && !Path.GetFullPath(source).Equals(Path.GetFullPath(normalized), StringComparison.OrdinalIgnoreCase)) try { if (Directory.Exists(directory)) Directory.Delete(directory, true); } catch { } } +} diff --git a/DocumentAutomation/src/DocxWorkbench.WordInterop/TemplateDocumentGenerator.cs b/DocumentAutomation/src/DocxWorkbench.WordInterop/TemplateDocumentGenerator.cs new file mode 100644 index 0000000..af23ea9 --- /dev/null +++ b/DocumentAutomation/src/DocxWorkbench.WordInterop/TemplateDocumentGenerator.cs @@ -0,0 +1,49 @@ +using System.IO.Compression; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Xml.Linq; +using DocumentAutomation.Shared; +using DocxWorkbench.Core; + +namespace DocxWorkbench.WordInterop; + +public sealed class TemplateDocumentGenerator : ITemplateDocumentGenerator, IDisposable +{ + private readonly ITemplatePackageStore _packages; + private readonly IWordDocumentNormalizer _normalizer; + private readonly WordDocumentService _word = new(); + private readonly IDocumentInspector _inspector; + public TemplateDocumentGenerator(ITemplatePackageStore packages, IWordDocumentNormalizer normalizer, IDocumentInspector inspector) { _packages = packages; _normalizer = normalizer; _inspector = inspector; } + + public async Task GenerateAsync(TemplateGenerationRequest request, IProgress? progress, CancellationToken cancellationToken) + { + var opened = _packages.Open(request.TemplatePackagePath); DocumentNormalizationResult? normalizedBody = null; var outputPdf = Path.ChangeExtension(request.OutputDocxPath, ".pdf"); + try + { + var bodyPath = request.ContentSourcePath; + if (Path.GetExtension(bodyPath).Equals(".doc", StringComparison.OrdinalIgnoreCase)) { normalizedBody = await _normalizer.NormalizeAsync(new DocumentNormalizationRequest { SourcePath = bodyPath, WorkingDirectory = Path.Combine(opened.WorkingDirectory, "body") }, progress, cancellationToken); bodyPath = normalizedBody.NormalizedPath; } + var workingTemplate = Path.Combine(opened.WorkingDirectory, "working-template.docx"); File.Copy(opened.TemplateDocxPath, workingTemplate, true); ReplaceScalarTokens(workingTemplate, request.ScalarValues); progress?.Report(new WordOperationProgress { Stage = "Generate", Message = "Building document from learned template" }); + var result = await _word.BuildDocumentAsync(new WordDocumentBuildRequest { TemplatePath = workingTemplate, BodyPath = bodyPath, OutputDocxPath = request.OutputDocxPath }, progress is null ? null : new Progress(x => progress.Report(new WordOperationProgress { Stage = "Generate", Message = x })), cancellationToken); await _word.ExportPdfAsync(result.OutputDocxPath, outputPdf, cancellationToken); + var validation = Validate(result.OutputDocxPath, opened.Template.TemplateId); var validationPath = Path.ChangeExtension(request.OutputDocxPath, ".validation.json"); File.WriteAllText(validationPath, JsonSerializer.Serialize(validation, new JsonSerializerOptions { WriteIndented = true })); return new TemplateGenerationResult { OutputDocxPath = result.OutputDocxPath, OutputPdfPath = outputPdf, GenerationLogPath = result.LogPath, ValidationLogPath = validationPath, Validation = validation }; + } + finally { TryDeleteDirectory(opened.WorkingDirectory); } + } + + private IReadOnlyList Validate(string output, string templateId) + { + var inspection = _inspector.Inspect(output); var result = new List { new() { RuleId = "DOCX_EXISTS", Status = File.Exists(output) ? RuleEvaluationStatus.Pass : RuleEvaluationStatus.Fail, Message = "Output DOCX exists.", DocumentLocation = output }, new() { RuleId = "NO_BODY_MARKER", Status = inspection.InsertionPoint.Type == "None" ? RuleEvaluationStatus.Pass : RuleEvaluationStatus.NeedsReview, Message = inspection.InsertionPoint.Type == "None" ? "No unresolved insertion marker remains." : "An insertion marker remains in the output.", DocumentLocation = "word/document.xml" }, new() { RuleId = "NO_TRACKED_CHANGES", Status = inspection.HasTrackedChanges ? RuleEvaluationStatus.Fail : RuleEvaluationStatus.Pass, Message = inspection.HasTrackedChanges ? "Tracked changes remain." : "No tracked changes detected." }, new() { RuleId = "TEMPLATE_ID", Status = RuleEvaluationStatus.Pass, Message = $"Generated from template {templateId}." } }; return result; + } + + private static void ReplaceScalarTokens(string path, IReadOnlyDictionary values) + { + if (values.Count == 0) return; using var archive = ZipFile.Open(path, ZipArchiveMode.Update); var w = XNamespace.Get("http://schemas.openxmlformats.org/wordprocessingml/2006/main"); + foreach (var entry in archive.Entries.Where(x => x.FullName.StartsWith("word/", StringComparison.OrdinalIgnoreCase) && x.FullName.EndsWith(".xml", StringComparison.OrdinalIgnoreCase)).ToArray()) + { + XDocument document; using (var reader = new StreamReader(entry.Open())) document = XDocument.Parse(reader.ReadToEnd(), LoadOptions.PreserveWhitespace); var changed = false; foreach (var text in document.Descendants(w + "t")) { var value = text.Value; foreach (var pair in values) { var token = "{{" + pair.Key + "}}"; if (!value.Contains(token, StringComparison.OrdinalIgnoreCase)) continue; value = value.Replace(token, pair.Value, StringComparison.OrdinalIgnoreCase); changed = true; } text.Value = value; } if (!changed) continue; entry.Delete(); var replacement = archive.CreateEntry(entry.FullName, CompressionLevel.Optimal); using var stream = replacement.Open(); using var writer = new StreamWriter(stream, new UTF8Encoding(false)); document.Save(writer, SaveOptions.DisableFormatting); + } + } + + private static void TryDeleteDirectory(string path) { try { if (Directory.Exists(path)) Directory.Delete(path, true); } catch { } } + public void Dispose() { _word.Dispose(); if (_normalizer is IDisposable disposable) disposable.Dispose(); } +} diff --git a/DocumentAutomation/src/DocxWorkbench.WordInterop/WordDocumentNormalizer.cs b/DocumentAutomation/src/DocxWorkbench.WordInterop/WordDocumentNormalizer.cs new file mode 100644 index 0000000..95d9c01 --- /dev/null +++ b/DocumentAutomation/src/DocxWorkbench.WordInterop/WordDocumentNormalizer.cs @@ -0,0 +1,42 @@ +using System.Diagnostics; +using System.Runtime.InteropServices; +using System.Security.Cryptography; +using DocumentAutomation.Shared; +using DocxWorkbench.Core; + +namespace DocxWorkbench.WordInterop; + +public sealed class WordDocumentNormalizer : IWordDocumentNormalizer, IDisposable +{ + private readonly StaWordQueue _queue; + public WordDocumentNormalizer() : this(new StaWordQueue(), true) { } + internal WordDocumentNormalizer(StaWordQueue queue) : this(queue, false) { } + private readonly bool _ownsQueue; + private WordDocumentNormalizer(StaWordQueue queue, bool ownsQueue) { _queue = queue; _ownsQueue = ownsQueue; } + + public Task NormalizeAsync(DocumentNormalizationRequest request, IProgress? progress, CancellationToken cancellationToken) + => _queue.Enqueue(() => NormalizeOnSta(request, progress, cancellationToken), cancellationToken); + + private static DocumentNormalizationResult NormalizeOnSta(DocumentNormalizationRequest request, IProgress? progress, CancellationToken token) + { + if (!File.Exists(request.SourcePath)) throw new FileNotFoundException("Document source was not found.", request.SourcePath); + var extension = Path.GetExtension(request.SourcePath); + if (!extension.Equals(".doc", StringComparison.OrdinalIgnoreCase) && !extension.Equals(".docx", StringComparison.OrdinalIgnoreCase)) throw new UnsupportedDocumentFormatException("Template source must be DOC or DOCX."); + var started = Stopwatch.StartNew(); var sourceHash = Hash(request.SourcePath); var work = Path.GetFullPath(request.WorkingDirectory); Directory.CreateDirectory(work); var normalized = Path.Combine(work, "normalized.docx"); token.ThrowIfCancellationRequested(); + if (extension.Equals(".docx", StringComparison.OrdinalIgnoreCase)) { File.Copy(request.SourcePath, normalized, true); return new DocumentNormalizationResult { SourcePath = request.SourcePath, NormalizedPath = normalized, SourceSha256 = sourceHash, NormalizedSha256 = Hash(normalized), WasConverted = false, Duration = started.Elapsed }; } + dynamic? application = null; dynamic? documents = null; dynamic? document = null; var proxies = new List(); + try + { + progress?.Report(new WordOperationProgress { Stage = "NormalizeDoc", Message = "Opening binary DOC read-only" }); var wordType = Type.GetTypeFromProgID("Word.Application", false) ?? throw new WordNotInstalledException("Microsoft Word is not installed or is not registered in COM."); application = Own(Activator.CreateInstance(wordType), proxies) ?? throw new WordAutomationException("Could not create Word.Application."); application.Visible = false; application.DisplayAlerts = 0; if (request.DisableMacros) { try { application.AutomationSecurity = 3; } catch { } } documents = Own(application.Documents, proxies); document = Own(documents.Open(request.SourcePath, false, true, false), proxies) ?? throw new WordAutomationException("Word could not open the DOC source read-only."); token.ThrowIfCancellationRequested(); progress?.Report(new WordOperationProgress { Stage = "NormalizeDoc", Message = "Saving temporary DOCX" }); document.SaveAs2(normalized, 12); int? pageCount = null; try { document.Repaginate(); pageCount = (int)document.ComputeStatistics(2); } catch { } document.Close(0); document = null; return new DocumentNormalizationResult { SourcePath = request.SourcePath, NormalizedPath = normalized, SourceSha256 = sourceHash, NormalizedSha256 = Hash(normalized), WasConverted = true, Duration = started.Elapsed, PageCount = pageCount }; + } + catch (WordNotInstalledException) { throw; } + catch (WordAutomationException) { throw; } + catch (Exception ex) { throw new WordAutomationException("DOC normalization failed.", ex); } + finally { try { if (document is not null) document.Close(0); } catch { } try { if (application is not null) application.Quit(0); } catch { } ReleaseAll(proxies); } + } + + private static string Hash(string path) => Convert.ToHexString(SHA256.HashData(File.ReadAllBytes(path))); + private static T? Own(T? value, List proxies) where T : class { if (value is not null) proxies.Add(value); return value; } + private static void ReleaseAll(IEnumerable proxies) { foreach (var value in proxies.Reverse()) try { if (value is not null && Marshal.IsComObject(value)) Marshal.FinalReleaseComObject(value); } catch { } } + public void Dispose() { if (_ownsQueue) _queue.Dispose(); } +} diff --git a/DocumentAutomation/src/DocxWorkbench.WordInterop/WordDocumentService.cs b/DocumentAutomation/src/DocxWorkbench.WordInterop/WordDocumentService.cs index de4cf23..0f2c4a1 100644 --- a/DocumentAutomation/src/DocxWorkbench.WordInterop/WordDocumentService.cs +++ b/DocumentAutomation/src/DocxWorkbench.WordInterop/WordDocumentService.cs @@ -1,6 +1,7 @@ using System.Diagnostics; using System.IO.Compression; using System.Runtime.InteropServices; +using System.Security.Cryptography; using System.Text.Json; using System.Text; using System.Text.RegularExpressions; @@ -40,7 +41,7 @@ public sealed class WordDocumentService : IWordDocumentService, IDisposable range = Own(FindInsertionRange(document, insertion.Type, proxies), proxies) ?? throw new WordAutomationException("The insertion point disappeared while opening the template."); token.ThrowIfCancellationRequested(); progress?.Report($"Inserting {Path.GetExtension(request.BodyPath).ToUpperInvariant()} body"); if (request.BodyPath.EndsWith(".txt", StringComparison.OrdinalIgnoreCase)) range.Text = File.ReadAllText(request.BodyPath).Replace("\r\n", "\r").Replace('\n', '\r'); else bodyImport = _bodyImporter.ImportOnSta(application, document, range, new DocxBodyImportRequest { SourcePath = request.BodyPath, DestinationRangeHandle = range }, progress is null ? null : new Progress(x => progress.Report(x.Message ?? x.Stage)), token, proxies); token.ThrowIfCancellationRequested(); progress?.Report("Updating fields and pagination"); document.Repaginate(); document.Fields.Update(); UpdateHeadersFooters(document, proxies); document.SaveAs2(stagedOutput, 12); document.Close(0); document = null; RemoveUnreferencedHeaderFooterParts(stagedOutput); CommitOutput(stagedOutput, request.OutputDocxPath); - var logPath = Path.ChangeExtension(request.OutputDocxPath, ".generation.json"); File.WriteAllText(logPath, JsonSerializer.Serialize(new { operationId = Guid.NewGuid(), template = request.TemplatePath, body = request.BodyPath, output = request.OutputDocxPath, insertionPoint = insertion.Type, wordPid, staThread = Thread.CurrentThread.ManagedThreadId, staApartment = Thread.CurrentThread.GetApartmentState().ToString(), durationMs = started.ElapsedMilliseconds, completedAtUtc = DateTime.UtcNow }, new JsonSerializerOptions { WriteIndented = true })); + var logPath = Path.ChangeExtension(request.OutputDocxPath, ".generation.json"); File.WriteAllText(logPath, JsonSerializer.Serialize(new { operationId = Guid.NewGuid(), operationType = "BuildDocument", startedAtUtc = DateTime.UtcNow.Subtract(started.Elapsed), completedAtUtc = DateTime.UtcNow, template = request.TemplatePath, templateSha256 = Hash(request.TemplatePath), templateVersion = "1.0", body = request.BodyPath, sourceSha256 = Hash(request.BodyPath), output = request.OutputDocxPath, resultSha256 = Hash(request.OutputDocxPath), insertionPoint = insertion.Type, wordPid, staThread = Thread.CurrentThread.ManagedThreadId, staApartment = Thread.CurrentThread.GetApartmentState().ToString(), durationMs = started.ElapsedMilliseconds, status = "Succeeded", errorCode = (string?)null }, new JsonSerializerOptions { WriteIndented = true })); return new WordDocumentResult { OutputDocxPath = request.OutputDocxPath, LogPath = logPath, Duration = started.Elapsed, InsertionPoint = insertion.Type, BodyImport = bodyImport }; } catch (WordNotInstalledException) { throw; } @@ -81,6 +82,7 @@ public sealed class WordDocumentService : IWordDocumentService, IDisposable private static int? TryGetPid(dynamic application) { try { return Process.GetProcessById((int)application.Hwnd).Id; } catch { return null; } } private static T? Own(T? value, List proxies) where T : class { if (value is not null) proxies.Add(value); return value; } private static void ReleaseAll(IEnumerable proxies) { foreach (var proxy in proxies.Reverse()) try { if (proxy is not null && Marshal.IsComObject(proxy)) Marshal.FinalReleaseComObject(proxy); } catch { } } + private static string Hash(string path) => Convert.ToHexString(SHA256.HashData(File.ReadAllBytes(path))); private static void CommitOutput(string staged, string output) { var full = Path.GetFullPath(output); if (File.Exists(full)) File.Copy(full, full + ".bak-" + DateTime.UtcNow.ToString("yyyyMMddHHmmss"), true); File.Copy(staged, full, true); } private static void RemoveUnreferencedHeaderFooterParts(string path) { diff --git a/DocumentAutomation/tests/DocumentAutomation.IntegrationTests/DocumentAutomation.IntegrationTests.csproj b/DocumentAutomation/tests/DocumentAutomation.IntegrationTests/DocumentAutomation.IntegrationTests.csproj index 9402064..b2f4d91 100644 --- a/DocumentAutomation/tests/DocumentAutomation.IntegrationTests/DocumentAutomation.IntegrationTests.csproj +++ b/DocumentAutomation/tests/DocumentAutomation.IntegrationTests/DocumentAutomation.IntegrationTests.csproj @@ -9,7 +9,7 @@ true - + diff --git a/DocumentAutomation/tests/DocumentAutomation.IntegrationTests/WordComIntegrationTests.cs b/DocumentAutomation/tests/DocumentAutomation.IntegrationTests/WordComIntegrationTests.cs index cdbf93f..03e6d6e 100644 --- a/DocumentAutomation/tests/DocumentAutomation.IntegrationTests/WordComIntegrationTests.cs +++ b/DocumentAutomation/tests/DocumentAutomation.IntegrationTests/WordComIntegrationTests.cs @@ -4,6 +4,7 @@ using DocumentFormat.OpenXml.Packaging; using DocumentFormat.OpenXml.Wordprocessing; using DocumentAutomation.Shared; using DocxWorkbench.WordInterop; +using DocxWorkbench.OpenXml; using Xunit.Sdk; namespace DocumentAutomation.IntegrationTests; @@ -47,6 +48,18 @@ public sealed class WordComIntegrationTests await RequireWord(); var template = CreateDocx(true); var source = CreateComplexSource(); var output = Path.Combine(Path.GetTempPath(), Guid.NewGuid() + ".docx"); try { using var service = new WordDocumentService(); var result = await service.BuildDocumentAsync(new WordDocumentBuildRequest { TemplatePath = template, BodyPath = source, OutputDocxPath = output }, null, CancellationToken.None); using var document = WordprocessingDocument.Open(result.OutputDocxPath, false); Assert.Single(document.MainDocumentPart!.Document.Descendants()); Assert.Contains("DOCX complex body", document.MainDocumentPart.Document.InnerText); } finally { File.Delete(template); File.Delete(source); File.Delete(output); File.Delete(Path.ChangeExtension(output, ".generation.json")); } } + [Fact] + public async Task TemplateLearning_EndToEnd_RealDocToDtemplateToDocxAndPdf() + { + await RequireWord(); var source = @"C:\Users\codex\Desktop\НХС-33-УК3-ОЗХ-13011-000-НК-ОД.doc"; var content = @"C:\Users\codex\Desktop\НХС-33-УК3-ОЗХ-13011-000-НК-ОД.docx"; if (!File.Exists(source) || !File.Exists(content)) throw SkipException.ForSkip("The supplied real DOC/DOCX fixtures are not available."); + var package = Path.Combine(Path.GetTempPath(), Guid.NewGuid() + ".dtemplate"); var output = Path.Combine(Path.GetTempPath(), Guid.NewGuid() + ".docx"); var sourceHash = Convert.ToHexString(System.Security.Cryptography.SHA256.HashData(File.ReadAllBytes(source))); var before = ProcessIds(); + try + { + using var normalizer = new WordDocumentNormalizer(); var learning = new TemplateLearningService(normalizer, new TemplatePackageStore(), new OpenXmlDocumentInspector()); var learned = await learning.LearnAsync(new TemplateLearningRequest { SourcePath = source, TemplateName = "Real sample", OutputPackagePath = package }, null, CancellationToken.None); Assert.NotEmpty(learned.Slots); Assert.True(File.Exists(package)); using var generator = new TemplateDocumentGenerator(new TemplatePackageStore(), new WordDocumentNormalizer(), new OpenXmlDocumentInspector()); var result = await generator.GenerateAsync(new TemplateGenerationRequest { TemplatePackagePath = package, ContentSourcePath = content, OutputDocxPath = output }, null, CancellationToken.None); Assert.True(File.Exists(result.OutputDocxPath)); Assert.True(File.Exists(result.OutputPdfPath)); Assert.Equal(sourceHash, Convert.ToHexString(System.Security.Cryptography.SHA256.HashData(File.ReadAllBytes(source)))); Assert.Empty(ProcessIds().Except(before)); + } + finally { File.Delete(package); File.Delete(output); File.Delete(Path.ChangeExtension(output, ".pdf")); File.Delete(Path.ChangeExtension(output, ".generation.json")); File.Delete(Path.ChangeExtension(output, ".validation.json")); } + } + private static async Task RequireWord() { if (!OperatingSystem.IsWindows() || Type.GetTypeFromProgID("Word.Application", false) is null) throw SkipException.ForSkip("Microsoft Word COM is not installed or registered on this workstation."); await Task.CompletedTask; } private static async Task BuildTxt() { var template = CreateDocx(true); var body = Path.Combine(Path.GetTempPath(), Guid.NewGuid() + ".txt"); var output = Path.Combine(Path.GetTempPath(), Guid.NewGuid() + ".docx"); File.WriteAllText(body, "TXT integration body"); try { using var service = new WordDocumentService(); return await service.BuildDocumentAsync(new WordDocumentBuildRequest { TemplatePath = template, BodyPath = body, OutputDocxPath = output }, null, CancellationToken.None); } finally { File.Delete(template); File.Delete(body); } } private static HashSet ProcessIds() => Process.GetProcessesByName("WINWORD").Select(x => x.Id).ToHashSet(); diff --git a/DocumentAutomation/tests/DocxWorkbench.Tests/DocxWorkbench.Tests.csproj b/DocumentAutomation/tests/DocxWorkbench.Tests/DocxWorkbench.Tests.csproj index a2aa407..56974d1 100644 --- a/DocumentAutomation/tests/DocxWorkbench.Tests/DocxWorkbench.Tests.csproj +++ b/DocumentAutomation/tests/DocxWorkbench.Tests/DocxWorkbench.Tests.csproj @@ -9,7 +9,7 @@ true - + diff --git a/DocumentAutomation/tests/DocxWorkbench.Tests/TemplateLearningTests.cs b/DocumentAutomation/tests/DocxWorkbench.Tests/TemplateLearningTests.cs new file mode 100644 index 0000000..bcaee35 --- /dev/null +++ b/DocumentAutomation/tests/DocxWorkbench.Tests/TemplateLearningTests.cs @@ -0,0 +1,25 @@ +using System.IO.Compression; +using System.Security.Cryptography; +using DocumentAutomation.Shared; +using DocxWorkbench.OpenXml; +using DocxWorkbench.WordInterop; + +namespace DocxWorkbench.Tests; + +public sealed class TemplateLearningTests +{ + [Fact] + public async Task NormalizeDocx_UsesSafeWorkingCopy() + { + var source = CreateDocx("template"); var hash = Hash(source); var working = Path.Combine(Path.GetTempPath(), "learning-test-" + Guid.NewGuid().ToString("N")); using var normalizer = new WordDocumentNormalizer(); try { var result = await normalizer.NormalizeAsync(new DocumentNormalizationRequest { SourcePath = source, WorkingDirectory = working }, null, CancellationToken.None); Assert.False(result.WasConverted); Assert.NotEqual(Path.GetFullPath(source), Path.GetFullPath(result.NormalizedPath)); Assert.Equal(hash, Hash(source)); Assert.Equal(hash, result.SourceSha256); Assert.True(File.Exists(result.NormalizedPath)); } finally { File.Delete(source); if (Directory.Exists(working)) Directory.Delete(working, true); } + } + + [Fact] + public async Task TemplatePackage_CanBeCreatedAndReopened() + { + var source = CreateDocx("{{DOC_BODY}}"); var package = Path.Combine(Path.GetTempPath(), Guid.NewGuid() + ".dtemplate"); using var normalizer = new WordDocumentNormalizer(); try { var service = new TemplateLearningService(normalizer, new TemplatePackageStore()); var learned = await service.LearnAsync(new TemplateLearningRequest { SourcePath = source, TemplateName = "Unit template", OutputPackagePath = package }, null, CancellationToken.None); Assert.NotEmpty(learned.Slots); Assert.Contains(learned.PageArchetypes, x => x.IsFirstPage); using var archive = ZipFile.OpenRead(package); Assert.Contains(archive.Entries, x => x.FullName == "manifest.json"); Assert.Contains(archive.Entries, x => x.FullName == "template.docx"); var opened = new TemplatePackageStore().Open(package); Assert.Equal(learned.TemplateId, opened.Template.TemplateId); Assert.True(File.Exists(opened.TemplateDocxPath)); Directory.Delete(opened.WorkingDirectory, true); } finally { File.Delete(source); File.Delete(package); } + } + + private static string CreateDocx(string text) { var path = Path.Combine(Path.GetTempPath(), Guid.NewGuid() + ".docx"); using var document = DocumentFormat.OpenXml.Packaging.WordprocessingDocument.Create(path, DocumentFormat.OpenXml.WordprocessingDocumentType.Document); var main = document.AddMainDocumentPart(); main.Document = new DocumentFormat.OpenXml.Wordprocessing.Document(new DocumentFormat.OpenXml.Wordprocessing.Body(new DocumentFormat.OpenXml.Wordprocessing.Paragraph(new DocumentFormat.OpenXml.Wordprocessing.Run(new DocumentFormat.OpenXml.Wordprocessing.Text(text))))); main.Document.Save(); return path; } + private static string Hash(string path) => Convert.ToHexString(SHA256.HashData(File.ReadAllBytes(path))); +}