Semantic markup and ICS table generation

.ci/asciidoc-converter/build_document_pdf.bat

@@ -0,0 +1,3 @@
+gradlew.bat run --args="--input-file ../../asciidoc/sdpi-supplement.adoc --output-folder ../../sdpi-supplement --backend pdf"
+
+

.ci/asciidoc-converter/build_test_result.bat

@@ -0,0 +1,7 @@
+:: Creates html output file based on a test ascii doc. 
+:: We leave off all the html headers to make the test more robust 
+:: (that is, test don't depend on css in headers). 
+
+:: Argument: name of test ascii doc file in src/test/resources/
+
+gradlew.bat run --args="--input-file src/test/resources/%1.adoc --output-folder src/test/resources --backend html --test"

.ci/asciidoc-converter/src/main/kotlin/org/sdpi/AsciidocConverter.kt

@@ -6,73 +6,147 @@ import org.asciidoctor.Asciidoctor
 import org.asciidoctor.Options
 import org.asciidoctor.SafeMode
 import org.sdpi.asciidoc.extension.*
-import org.sdpi.asciidoc.github.Issues
 import java.io.File
 import java.io.OutputStream
 import java.nio.file.Files
 import java.nio.file.Path
+import kotlin.io.path.absolutePathString
+
+/**
+ * Options to configure AsciidocConverter.
+ */
+class ConverterOptions(
+    /**
+     * Token to access the GitHub api. For including issues
+     * in the document output, for example.
+     */
+    val githubToken: String? = null,
+
+    /**
+     * Defines the target format for the document output. See
+     * https://docs.asciidoctor.org/asciidoctorj/latest/asciidoctor-api-options/#backend
+     * Typically "html" or "pdf"
+     */
+    val outputFormat: String = "html",
+
+    /**
+     * When true, document structure is written to the log stream
+     * for diagnostics.
+     */
+    val dumpStructure: Boolean = false,
+
+    /**
+     * When true, the document output is simplified for unit
+     * tests. For example, the style sheet is not included.
+     */
+    val generateTestOutput: Boolean = false,
+
+    /**
+     * Folder where extracts (requirements, use-cases, etc.) should
+     * be placed. If null, the extracts won't be written.
+     */
+    val extractsFolder: Path? = null,
+) {
+    companion object {
+        private const val DEFAULT_EXTRACTS_FOLDER: String = "referenced-artifacts"
+
+        fun makeDefaultPath(strOutputFolder: String): Path {
+            return Path.of(strOutputFolder, DEFAULT_EXTRACTS_FOLDER)
+        }
+    }
+}
+
 
 class AsciidocConverter(
     private val inputType: Input,
-    private val outputFile: File,
-    private val githubToken: String?,
-    private val mode: Mode = Mode.Productive,
+    private val outputFile: OutputStream,
+    private val conversionOptions: ConverterOptions,
 ) : Runnable {
     override fun run() {
         val options = Options.builder()
             .safe(SafeMode.UNSAFE)
-            .backend(BACKEND)
+            .backend(conversionOptions.outputFormat)
             .sourcemap(true)
-            .toFile(outputFile).build()
+            .headerFooter(!conversionOptions.generateTestOutput)
+            .toStream(outputFile).build()
 
         val asciidoctor = Asciidoctor.Factory.create()
 
         val anchorReplacements = AnchorReplacementsMap()
 
-        val requirementsBlockProcessor = RequirementsBlockProcessor()
-        asciidoctor.javaExtensionRegistry().block(requirementsBlockProcessor)
-        asciidoctor.javaExtensionRegistry().treeprocessor(
-            NumberingProcessor(
-                when (mode) {
-                    is Mode.Test -> mode.structureDump
-                    else -> null
-                },
-                anchorReplacements
-            )
-        )
-        asciidoctor.javaExtensionRegistry().treeprocessor(RequirementLevelProcessor())
-        asciidoctor.javaExtensionRegistry().preprocessor(IssuesSectionPreprocessor(githubToken))
+        // Formats sdpi_requirement blocks & their content.
+        //   * RequirementBlockProcessor2 handles the containing sdpi_requirement block
+        //   * RelatedBlockProcessor handles [RELATED] blocks within requirement blocks.
+        //   * RequirementExampleBlockProcessor handles [EXAMPLE] blocks within requirement blocks.
+        asciidoctor.javaExtensionRegistry().block(RequirementBlockProcessor2())
+        asciidoctor.javaExtensionRegistry().block(RelatedBlockProcessor())
+        asciidoctor.javaExtensionRegistry().block(RequirementExampleBlockProcessor())
+
+        asciidoctor.javaExtensionRegistry().treeprocessor(NumberingProcessor(null, anchorReplacements))
+
+        // Gather bibliography entries.
+        val bibliographyCollector = BibliographyCollector()
+        asciidoctor.javaExtensionRegistry().treeprocessor(bibliographyCollector)
+
+        // Gather SDPI specific information from the document such as
+        // requirements and use-cases.
+        val infoCollector = TreeInfoCollector(bibliographyCollector)
+        asciidoctor.javaExtensionRegistry().treeprocessor(infoCollector)
+
+        // Support to insert tables of requirements etc. sdpi_requirement_table macros.
+        // Block macro processors insert placeholders that are populated when the tree is ready.
+        // Tree processors fill in the placeholders.
+        asciidoctor.javaExtensionRegistry().blockMacro(AddRequirementQueryPlaceholder())
+        asciidoctor.javaExtensionRegistry().blockMacro(AddICSPlaceholder())
+        asciidoctor.javaExtensionRegistry().treeprocessor(PopulateTables(infoCollector.info()))
+
+        // Handle inline macros to cross-reference information from the document tree.
+        asciidoctor.javaExtensionRegistry().inlineMacro(RequirementReferenceMacroProcessor(infoCollector.info()))
+        asciidoctor.javaExtensionRegistry().inlineMacro(UseCaseReferenceMacroProcessor(infoCollector.info()))
+
+        asciidoctor.javaExtensionRegistry().preprocessor(IssuesSectionPreprocessor(conversionOptions.githubToken))
         asciidoctor.javaExtensionRegistry().preprocessor(DisableSectNumsProcessor())
         asciidoctor.javaExtensionRegistry().preprocessor(ReferenceSanitizerPreprocessor(anchorReplacements))
-        asciidoctor.javaExtensionRegistry()
-            .postprocessor(ReferenceSanitizerPostprocessor(anchorReplacements))
+        if (conversionOptions.outputFormat == "html") {
+            // Post processor not supported for PDFs
+            // https://docs.asciidoctor.org/asciidoctorj/latest/extensions/postprocessor/
+            asciidoctor.javaExtensionRegistry().postprocessor(ReferenceSanitizerPostprocessor(anchorReplacements))
+        }
+
+        // Dumps tree of document structure to stdio.
+        // Best not to use for very large documents!
+        // Note: enabling this breaks variable replacement for {var_transaction_id}. Unclear why.
+        if (conversionOptions.dumpStructure) {
+            asciidoctor.javaExtensionRegistry().treeprocessor(DumpTreeInfo())
+        }
 
         asciidoctor.requireLibrary("asciidoctor-diagram") // enables plantuml
+
         when (inputType) {
             is Input.FileInput -> asciidoctor.convertFile(inputType.file, options)
             is Input.StringInput -> asciidoctor.convert(inputType.string, options)
         }
 
-        asciidoctor.shutdown()
+        if (conversionOptions.extractsFolder != null) {
+            val jsonFormatter = Json { prettyPrint = true }
 
-        val referencedArtifactsName = "referenced-artifacts"
-        val path = Path.of(outputFile.parentFile.absolutePath, referencedArtifactsName)
-        Files.createDirectories(path)
-        val reqsDump = Json.encodeToString(requirementsBlockProcessor.detectedRequirements())
-        Path.of(path.toFile().absolutePath, "sdpi-requirements.json").toFile().writeText(reqsDump)
+            writeArtifact("sdpi-requirements", jsonFormatter.encodeToString(infoCollector.info().requirements()))
+            writeArtifact("sdpi-use-cases", jsonFormatter.encodeToString(infoCollector.info().useCases()))
+        }
+
+        asciidoctor.shutdown()
     }
 
-    private companion object {
-        const val BACKEND = "html"
+    private fun writeArtifact(strArtifactName: String, strArtifact: String) {
+        if (conversionOptions.extractsFolder != null) {
+            Files.createDirectories(conversionOptions.extractsFolder)
+            Path.of(conversionOptions.extractsFolder.absolutePathString(), "${strArtifactName}.json").toFile()
+                .writeText(strArtifact)
+        }
     }
 
     sealed interface Input {
         data class FileInput(val file: File) : Input
         data class StringInput(val string: String) : Input
     }
-
-    sealed interface Mode {
-        object Productive : Mode
-        data class Test(val structureDump: OutputStream) : Mode
-    }
 }

.ci/asciidoc-converter/src/main/kotlin/org/sdpi/ConvertAndVerifySupplement.kt

@@ -1,19 +1,16 @@
 package org.sdpi
 
 import com.github.ajalt.clikt.core.CliktCommand
-import com.github.ajalt.clikt.parameters.options.default
-import com.github.ajalt.clikt.parameters.options.option
-import com.github.ajalt.clikt.parameters.options.required
-import com.github.ajalt.clikt.parameters.options.validate
+import com.github.ajalt.clikt.parameters.options.*
 import com.github.ajalt.clikt.parameters.types.choice
 import com.github.ajalt.clikt.parameters.types.file
 import org.apache.logging.log4j.kotlin.Logging
 import org.sdpi.asciidoc.AsciidocErrorChecker
-import org.sdpi.asciidoc.github.IssueImport
 import java.io.File
 import kotlin.system.exitProcess
 
-fun main(args: Array<String>) = ConvertAndVerifySupplement().main(args
+fun main(args: Array<String>) = ConvertAndVerifySupplement().main(
+    args
 //    when (System.getenv().containsKey("CI")) {
 //        true -> args.firstOrNull()?.split(" ") ?: listOf() // caution: blanks in quotes not covered here!
 //        false -> args.toList()
@@ -47,6 +44,12 @@ class ConvertAndVerifySupplement : CliktCommand("convert-supplement") {
 
     private val githubToken by option("--github-token", help = "Github token to request issues")
 
+    private val dumpStructure by option("--dump-structure", help = "Writes document tree to std-out during processing")
+        .flag(default = false)
+
+    private val testGenerator by option("--test", help = "Writes document without headers for test output")
+        .flag(default = false)
+
     override fun run() {
         runCatching {
             val asciidocErrorChecker = AsciidocErrorChecker()
@@ -59,7 +62,17 @@ class ConvertAndVerifySupplement : CliktCommand("convert-supplement") {
 
             logger.info { "Write output to '${outFile.canonicalPath}'" }
 
-            AsciidocConverter(AsciidocConverter.Input.FileInput(adocInputFile), outFile, githubToken).run()
+            AsciidocConverter(
+                AsciidocConverter.Input.FileInput(adocInputFile),
+                outFile.outputStream(),
+                ConverterOptions(
+                    githubToken = githubToken,
+                    extractsFolder = ConverterOptions.makeDefaultPath(outputFolder.absolutePath),
+                    outputFormat = backend,
+                    dumpStructure = dumpStructure,
+                    generateTestOutput = testGenerator,
+                )
+            ).run()
 
             asciidocErrorChecker.run()
 

.ci/asciidoc-converter/src/main/kotlin/org/sdpi/asciidoc/BlockAttribute.kt

@@ -7,7 +7,62 @@ enum class BlockAttribute(val key: String) {
     ID("id"),
     TITLE("title"),
     ROLE("role"),
-    REQUIREMENT_LEVEL("sdpi_req_level"),
     MAX_OCCURRENCE("sdpi_max_occurrence"),
-    VOLUME_CAPTION("sdpi_volume_caption")
+    VOLUME_CAPTION("sdpi_volume_caption"),
+
 }
+
+/**
+ * Defines attribute keywords for sdpi_requirement blocks
+ */
+sealed class RequirementAttributes {
+    enum class Common(val key: String) {
+        // Type of requirement.
+        TYPE("sdpi_req_type"),
+
+        // Requirement level (e.g., shall, may, etc)
+        LEVEL("sdpi_req_level"),
+
+        // Identifies a specification that the requirement belongs
+        // to. Used, for example, to determine the base oid for
+        // assigning globally unique object ids.
+        SPECIFICATION("sdpi_req_specification"),
+
+        // Groups requirement belongs to. Comma separated list.
+        GROUPS("sdpi_req_group"),
+
+    }
+
+    enum class UseCase(val key: String) {
+        // Attribute that identifies the use case associated with a
+        // USE_CASE requirement
+        ID("sdpi_use_case_id")
+    }
+
+    enum class RefIcs(val key: String) {
+        // The id of the reference to the standard containing the requirement
+        ID("sdpi_ref_id"),
+
+        // Section in the reference standard containing the requirement (e.g., "§6.2")
+        SECTION("sdpi_ref_section"),
+
+        // Requirement identifier in the referenced standard (e.g., "R1001")
+        REQUIREMENT("sdpi_ref_req")
+    }
+
+    enum class RiskMitigation(val key: String) {
+        // Type of risk (e.g., general, safety, effectiveness, etc)
+        SES_TYPE("sdpi_ses_type"),
+
+        // How requirement can be tested (e.g., inspection, interoperability, etc.).
+        TESTABILITY("sdpi_ses_test")
+    }
+}
+
+enum class UseCaseAttributes(val key: String) {
+    // Attribute to define an id for a use case section.
+    ID("sdpi_use_case_id"),
+
+    // Attribute to define an id for a use case scenario.
+    SCENARIO("sdpi_use_case_scenario")
+}

.ci/asciidoc-converter/src/main/kotlin/org/sdpi/asciidoc/Util.kt

@@ -56,11 +56,40 @@ fun StructuralNode.isAppendix() = when (val section = this.toSealed()) {
     else -> false
 }
 
+fun parseRequirementNumber(strRequirement: String): Int? {
+    val requirementParser = "^r(\\d+)$".toRegex()
+    val matchResults = requirementParser.findAll(strRequirement)
+    if (matchResults.any()) {
+        return matchResults.map { it.groupValues[1] }.toList().first().toInt()
+    }
+
+    return null
+}
+
 /**
  * Takes a string and converts it to a [RequirementLevel] enum.
  *
  * @param raw Raw text being shall, should or may.
  *
  * @return the [RequirementLevel] enum or null if the conversion failed (raw was not shall, should or may).
  */
-fun resolveRequirementLevel(raw: String) = RequirementLevel.values().firstOrNull { it.keyword == raw }
+fun resolveRequirementLevel(raw: String) = RequirementLevel.entries.firstOrNull { it.keyword == raw }
+
+/**
+ * Converts an object, typically from an attribute map, into
+ * a list of groups.
+ */
+fun getRequirementGroups(oGroups: Any?): List<String> {
+    if (oGroups == null) {
+        return listOf()
+    }
+    return oGroups.toString().split(",")
+}
+
+fun getLocation(block: StructuralNode): String {
+    return if (block.sourceLocation != null) {
+        "${block.sourceLocation.path}:${block.sourceLocation.lineNumber}"
+    } else {
+        "";
+    }
+}