doc: onboarding wiki article and general improvements (#1590)

* doc: onboarding wiki article and general improvements

* doc: fix-up permalinks

* doc: improve label function tsdoc

* doc: generate the FAQx

* wip(doc): some more faqs

* doc: finished up the faq

* refactor: cleaned up util directory

* feat: added setup:dev script

* docs: mention the setup script in the contributing article

* docs: whoops

Author: Ellpeck

.github/CONTRIBUTING.md

@@ -55,7 +55,7 @@ For more information on how to clone and work with a repository, please see [Git
 
 ## Git-Hooks
 
-Before you commit, please enable the project-specific git-hooks by running:
+Before you commit, please enable the project-specific git-hooks. You can do so through the `setup:dev` script as documented in the [onboarding wiki article](https://github.com/flowr-analysis/flowr/wiki/Onboarding), or through the following command:
 
 ```shell
 git config --local core.hooksPath .githooks/

.github/workflows/broken-links-and-wiki.yaml

@@ -101,6 +101,8 @@ jobs:
           update_page wiki/"Control Flow Graph" wiki:cfg
           update_page wiki/"Linting and Testing" wiki:linting-and-testing
           update_page wiki/"Core" wiki:core
+          update_page wiki/"Onboarding" wiki:onboarding
+          update_page wiki/"FAQ" wiki:faq
           update_page README gen:readme
           
           # check for an update in the versions! (including the updated name information)

package.json

@@ -12,6 +12,7 @@
     "url": "https://github.com/flowr-analysis/flowr/issues"
   },
   "scripts": {
+    "setup:dev": "git lfs fetch && npm ci && git config --local core.hooksPath .githooks/ && git push --dry-run",
     "main": "npm run build:bundle-flowr && node dist/src/cli/flowr.min.js",
     "flowr": "npm run main --",
     "main-dev": "ts-node src/cli/flowr.ts",
@@ -33,6 +34,8 @@
     "wiki:search-api": "ts-node src/documentation/print-search-wiki.ts",
     "wiki:linting-and-testing": "ts-node src/documentation/print-linting-and-testing-wiki.ts",
     "wiki:interface": "ts-node src/documentation/print-interface-wiki.ts",
+    "wiki:onboarding": "ts-node src/documentation/print-onboarding-wiki.ts",
+    "wiki:faq": "ts-node src/documentation/print-faq-wiki.ts",
     "wiki:cfg": "ts-node src/documentation/print-cfg-wiki.ts",
     "gen:readme": "ts-node src/documentation/print-readme.ts",
     "build": "tsc --project .",

src/benchmark/slicer.ts

@@ -13,7 +13,7 @@ import type { SliceResult } from '../slicing/static/slicer-types';
 import type { ReconstructionResult } from '../reconstruct/reconstruct';
 import type { PipelineExecutor } from '../core/pipeline-executor';
 import { guard } from '../util/assert';
-import { withoutWhitespace } from '../util/strings';
+import { withoutWhitespace } from '../util/text/strings';
 import type {
 	BenchmarkMemoryMeasurement,
 	CommonSlicerMeasurements,
@@ -45,7 +45,7 @@ import { TreeSitterExecutor } from '../r-bridge/lang-4.x/tree-sitter/tree-sitter
 import type { InGraphIdentifierDefinition } from '../dataflow/environments/identifier';
 import type { ContainerIndicesCollection } from '../dataflow/graph/vertex';
 import { isParentContainerIndex } from '../dataflow/graph/vertex';
-import { equidistantSampling } from '../util/arrays';
+import { equidistantSampling } from '../util/collections/arrays';
 
 /**
  * The logger to be used for benchmarking as a global object.

src/benchmark/summarizer/first-phase/input.ts

@@ -3,7 +3,7 @@ import fs from 'fs';
 import { processNextSummary, summarizeAllSummarizedStats } from '../second-phase/process';
 import { summarizeSlicerStats } from './process';
 import { guard } from '../../../util/assert';
-import { escape } from '../../../util/ansi';
+import { escape } from '../../../util/text/ansi';
 import { jsonReplacer } from '../../../util/json';
 import type { BenchmarkMemoryMeasurement, CommonSlicerMeasurements, PerSliceMeasurements, PerSliceStats, SlicerStats } from '../../stats/stats';
 import type { SlicingCriteria } from '../../../slicing/criterion/parse';

src/benchmark/summarizer/first-phase/process.ts

@@ -2,9 +2,9 @@ import * as tmp from 'tmp';
 import type { Reduction, SliceSizeCollection, SummarizedSlicerStats, TimePerToken } from '../data';
 
 import fs from 'fs';
-import { DefaultMap } from '../../../util/defaultmap';
+import { DefaultMap } from '../../../util/collections/defaultmap';
 import { log } from '../../../util/log';
-import { withoutWhitespace } from '../../../util/strings';
+import { withoutWhitespace } from '../../../util/text/strings';
 import type { SummarizedMeasurement } from '../../../util/summarizer';
 import { summarizeMeasurement } from '../../../util/summarizer';
 import { isNotUndefined } from '../../../util/assert';
@@ -14,7 +14,7 @@ import { RShell } from '../../../r-bridge/shell';
 import { retrieveNormalizedAstFromRCode, retrieveNumberOfRTokensOfLastParse } from '../../../r-bridge/retriever';
 import { visitAst } from '../../../r-bridge/lang-4.x/ast/model/processing/visitor';
 import { RType } from '../../../r-bridge/lang-4.x/ast/model/type';
-import { arraySum } from '../../../util/arrays';
+import { arraySum } from '../../../util/collections/arrays';
 
 const tempfile = (() => {
 	let _tempfile: tmp.FileResult | undefined = undefined;

src/benchmark/summarizer/second-phase/process.ts

@@ -1,6 +1,6 @@
 import type { Reduction, SummarizedSlicerStats, TimePerToken, UltimateSlicerStats } from '../data';
 import { summarizeSummarizedReductions, summarizeSummarizedMeasurement, summarizeSummarizedTimePerToken, summarizeTimePerToken } from '../first-phase/process';
-import { DefaultMap } from '../../../util/defaultmap';
+import { DefaultMap } from '../../../util/collections/defaultmap';
 import type { SummarizedMeasurement } from '../../../util/summarizer';
 import { summarizeMeasurement } from '../../../util/summarizer';
 import { guard } from '../../../util/assert';

src/benchmark/summarizer/summarizer.ts

@@ -13,7 +13,7 @@ import { Summarizer } from '../../util/summarizer';
 import { getAllFiles, readLineByLine, readLineByLineSync } from '../../util/files';
 import { jsonReplacer } from '../../util/json';
 import { ultimateStats2String } from '../stats/print';
-import { DefaultMap } from '../../util/defaultmap';
+import { DefaultMap } from '../../util/collections/defaultmap';
 import { log } from '../../util/log';
 
 export interface BenchmarkSummarizerConfiguration extends CommonSummarizerConfiguration {

src/cli/common/options.ts

@@ -1,6 +1,6 @@
 import type { OptionDefinition } from 'command-line-usage';
 import os from 'os';
-import { date2string } from '../../util/time';
+import { date2string } from '../../util/text/time';
 import { allFeatureNames } from '../../statistics/features/feature';
 
 /**

src/cli/common/script.ts

@@ -7,7 +7,7 @@ import { scripts } from './scripts-info';
 import commandLineUsage from 'command-line-usage';
 import type { CommonOptions } from './options';
 import commandLineArgs from 'command-line-args';
-import { italic } from '../../util/ansi';
+import { italic } from '../../util/text/ansi';
 import { log, LogLevel } from '../../util/log';
 
 /**

src/cli/flowr.ts

@@ -11,7 +11,7 @@ import { NetServer, WebSocketServerWrapper } from './repl/server/net';
 import { flowrVersion } from '../util/version';
 import commandLineUsage from 'command-line-usage';
 import { log, LogLevel } from '../util/log';
-import { bold, ColorEffect, Colors, FontStyles, formatter, italic, setFormatter, voidFormatter } from '../util/ansi';
+import { bold, ColorEffect, Colors, FontStyles, formatter, italic, setFormatter, voidFormatter } from '../util/text/ansi';
 import commandLineArgs from 'command-line-args';
 import type { EngineConfig, KnownEngines } from '../config';
 import { getConfig ,   amendConfig, getEngineConfig, parseConfig, setConfig, setConfigFile } from '../config';

src/cli/repl/commands/repl-cfg.ts

@@ -4,7 +4,7 @@ import { createDataflowPipeline } from '../../../core/steps/pipeline/default-pip
 import { fileProtocol, requestFromInput } from '../../../r-bridge/retriever';
 import { cfgToMermaid, cfgToMermaidUrl } from '../../../util/mermaid/cfg';
 import type { KnownParser } from '../../../r-bridge/parser';
-import { ColorEffect, Colors, FontStyles } from '../../../util/ansi';
+import { ColorEffect, Colors, FontStyles } from '../../../util/text/ansi';
 
 async function controlflow(parser: KnownParser, remainingLine: string) {
 	return await createDataflowPipeline(parser, {

src/cli/repl/commands/repl-commands.ts

@@ -13,9 +13,9 @@ import {
 	dataflowStarCommand
 } from './repl-dataflow';
 import { controlflowCommand, controlflowStarCommand } from './repl-cfg';
-import type { OutputFormatter } from '../../../util/ansi';
-import { italic , bold } from '../../../util/ansi';
-import { splitAtEscapeSensitive } from '../../../util/args';
+import type { OutputFormatter } from '../../../util/text/ansi';
+import { italic , bold } from '../../../util/text/ansi';
+import { splitAtEscapeSensitive } from '../../../util/text/args';
 import { guard } from '../../../util/assert';
 import { scripts } from '../../common/scripts-info';
 import { lineageCommand } from './repl-lineage';

src/cli/repl/commands/repl-dataflow.ts

@@ -3,7 +3,7 @@ import { createDataflowPipeline } from '../../../core/steps/pipeline/default-pip
 import { fileProtocol, requestFromInput } from '../../../r-bridge/retriever';
 import { graphToMermaid, graphToMermaidUrl } from '../../../util/mermaid/dfg';
 import type { KnownParser } from '../../../r-bridge/parser';
-import { ColorEffect, Colors, FontStyles } from '../../../util/ansi';
+import { ColorEffect, Colors, FontStyles } from '../../../util/text/ansi';
 
 /**
  * Obtain the dataflow graph using a known parser (such as the {@link RShell} or {@link TreeSitterExecutor}).

src/cli/repl/commands/repl-execute.ts

@@ -1,5 +1,5 @@
 import type { ReplCommand, ReplOutput } from './repl-main';
-import { ColorEffect, Colors, FontStyles, italic } from '../../../util/ansi';
+import { ColorEffect, Colors, FontStyles, italic } from '../../../util/text/ansi';
 import { RShell } from '../../../r-bridge/shell';
 import type { KnownParser } from '../../../r-bridge/parser';
 

src/cli/repl/commands/repl-main.ts

@@ -1,5 +1,5 @@
-import type { OutputFormatter } from '../../../util/ansi';
-import { formatter } from '../../../util/ansi';
+import type { OutputFormatter } from '../../../util/text/ansi';
+import { formatter } from '../../../util/text/ansi';
 import type { KnownParser } from '../../../r-bridge/parser';
 
 /**

src/cli/repl/commands/repl-normalize.ts

@@ -3,7 +3,7 @@ import { createNormalizePipeline } from '../../../core/steps/pipeline/default-pi
 import { fileProtocol, requestFromInput } from '../../../r-bridge/retriever';
 import { normalizedAstToMermaid, normalizedAstToMermaidUrl } from '../../../util/mermaid/ast';
 import type { KnownParser } from '../../../r-bridge/parser';
-import { ColorEffect, Colors, FontStyles } from '../../../util/ansi';
+import { ColorEffect, Colors, FontStyles } from '../../../util/text/ansi';
 
 async function normalize(parser: KnownParser, remainingLine: string) {
 	return await createNormalizePipeline(parser, {

src/cli/repl/commands/repl-parse.ts

@@ -1,6 +1,6 @@
 import type { ReplCommand } from './repl-main';
-import type { OutputFormatter } from '../../../util/ansi';
-import { FontStyles } from '../../../util/ansi';
+import type { OutputFormatter } from '../../../util/text/ansi';
+import { FontStyles } from '../../../util/text/ansi';
 import type { JsonEntry } from '../../../r-bridge/lang-4.x/ast/parser/json/format';
 import { convertPreparedParsedData , prepareParsedData } from '../../../r-bridge/lang-4.x/ast/parser/json/format';
 import {

src/cli/repl/commands/repl-query.ts

@@ -2,8 +2,8 @@ import type { DEFAULT_DATAFLOW_PIPELINE } from '../../../core/steps/pipeline/def
 import { createDataflowPipeline } from '../../../core/steps/pipeline/default-pipelines';
 import { fileProtocol, requestFromInput } from '../../../r-bridge/retriever';
 import type { ReplCommand, ReplOutput } from './repl-main';
-import { splitAtEscapeSensitive } from '../../../util/args';
-import { ansiFormatter, italic } from '../../../util/ansi';
+import { splitAtEscapeSensitive } from '../../../util/text/args';
+import { ansiFormatter, italic } from '../../../util/text/ansi';
 import { describeSchema } from '../../../util/schema';
 import type { Query, QueryResults, SupportedQueryTypes } from '../../../queries/query';
 import { AnyQuerySchema, QueriesSchema , executeQueries } from '../../../queries/query';

src/cli/repl/core.ts

@@ -9,8 +9,8 @@ import { tryExecuteRShellCommand } from './commands/repl-execute';
 import os from 'os';
 import path from 'path';
 import fs from 'fs';
-import { splitAtEscapeSensitive } from '../../util/args';
-import { FontStyles } from '../../util/ansi';
+import { splitAtEscapeSensitive } from '../../util/text/args';
+import { FontStyles } from '../../util/text/ansi';
 import { getCommand, getCommandNames } from './commands/repl-commands';
 import { getValidOptionsForCompletion, scripts } from '../common/scripts-info';
 import { fileProtocol } from '../../r-bridge/retriever';

src/cli/repl/prompt.ts

@@ -1,4 +1,4 @@
-import { ColorEffect, Colors, formatter } from '../../util/ansi';
+import { ColorEffect, Colors, formatter } from '../../util/text/ansi';
 
 export const rawPrompt = 'R>';
 // is a function as the 'formatter' is configured only after the cli options have been read

src/cli/repl/server/connection.ts

@@ -23,7 +23,7 @@ import type { PARSE_WITH_R_SHELL_STEP } from '../../../core/steps/all/core/00-pa
 import type { DataflowInformation } from '../../../dataflow/info';
 import type { NORMALIZE } from '../../../core/steps/all/core/10-normalize';
 import type { STATIC_DATAFLOW } from '../../../core/steps/all/core/20-dataflow';
-import { ansiFormatter, voidFormatter } from '../../../util/ansi';
+import { ansiFormatter, voidFormatter } from '../../../util/text/ansi';
 import { PipelineStepStage } from '../../../core/steps/pipeline-step';
 import { createSlicePipeline, DEFAULT_SLICING_PIPELINE } from '../../../core/steps/pipeline/default-pipelines';
 import type { Pipeline, PipelineOutput } from '../../../core/steps/pipeline/pipeline';

src/cli/script-core/statistics-core.ts

@@ -12,7 +12,7 @@ import { getStatsForSingleFile } from './statistics-helper-core';
 import commandLineArgs from 'command-line-args';
 import { scripts } from '../common/scripts-info';
 import type { StatsHelperCliOptions } from '../statistics-helper-app';
-import { setFormatter, voidFormatter } from '../../util/ansi';
+import { setFormatter, voidFormatter } from '../../util/text/ansi';
 
 const testRegex = /[^/]*\/test/i;
 const exampleRegex = /[^/]*\/example/i;

src/cli/script-core/statistics-helper-core.ts

@@ -12,10 +12,10 @@ import { STATIC_DATAFLOW } from '../../core/steps/all/core/20-dataflow';
 import { jsonReplacer } from '../../util/json';
 import { log } from '../../util/log';
 import { guard } from '../../util/assert';
-import { date2string } from '../../util/time';
+import { date2string } from '../../util/text/time';
 import type { StatsHelperCliOptions } from '../statistics-helper-app';
 import { create } from 'tar';
-import { setFormatter, voidFormatter } from '../../util/ansi';
+import { setFormatter, voidFormatter } from '../../util/text/ansi';
 
 function compressFolder(folder: string, target: string) {
 

src/control-flow/cfg-properties.ts

@@ -1,6 +1,6 @@
 import { CfgEdgeType, type ControlFlowGraph, type ControlFlowInformation } from './control-flow-graph';
 import type { NodeId } from '../r-bridge/lang-4.x/ast/model/processing/node-id';
-import { setMinus } from '../util/set';
+import { setMinus } from '../util/collections/set';
 import { log } from '../util/log';
 import { visitCfgInOrder, visitCfgInReverseOrder } from './simple-visitor';
 
@@ -109,4 +109,4 @@ export function assertCfgSatisfiesProperties(cfg: ControlFlowInformation, exclud
 		}
 	}
 	return true;
-}
+}

src/control-flow/diff-cfg.ts

@@ -6,7 +6,7 @@ import type { GenericDiffConfiguration } from '../util/diff';
 import { setDifference } from '../util/diff';
 import type { CfgEdge, CfgSimpleVertex, ControlFlowGraph } from './control-flow-graph';
 import { equalVertex } from './control-flow-graph';
-import { arrayEqual } from '../util/arrays';
+import { arrayEqual } from '../util/collections/arrays';
 
 
 /**

src/core/print/print.ts

@@ -1,5 +1,5 @@
 import type { IPipelineStep, StepProcessingFunction } from '../steps/pipeline-step';
-import type { TailOfArray } from '../../util/arrays';
+import type { TailOfArray } from '../../util/collections/arrays';
 import { guard } from '../../util/assert';
 
 /**

src/core/print/slice-diff-ansi.ts

@@ -1,7 +1,7 @@
 import type { SourceRange } from '../../util/range';
 import { mergeRanges, rangeCompare, rangesOverlap } from '../../util/range';
 import { isNotUndefined } from '../../util/assert';
-import { ansiFormatter, ColorEffect, Colors, FontStyles } from '../../util/ansi';
+import { ansiFormatter, ColorEffect, Colors, FontStyles } from '../../util/text/ansi';
 import type { NodeId } from '../../r-bridge/lang-4.x/ast/model/processing/node-id';
 import type { NormalizedAst } from '../../r-bridge/lang-4.x/ast/model/processing/decorate';
 

src/core/steps/pipeline/create-pipeline.ts

@@ -3,7 +3,7 @@ import { PipelineStepStage } from '../pipeline-step';
 import { InvalidPipelineError } from './invalid-pipeline-error';
 import type { Pipeline } from './pipeline';
 import { jsonReplacer } from '../../../util/json';
-import { partitionArray } from '../../../util/arrays';
+import { partitionArray } from '../../../util/collections/arrays';
 
 /**
  * Given a set of {@link IPipelineStep|steps} with their dependencies, this function verifies all requirements of {@link createPipeline}.
@@ -143,4 +143,3 @@ function initializeSteps(steps: readonly IPipelineStep[], stepMap: Map<PipelineS
 		}
 	}
 }
-

src/dataflow/graph/diff-dataflow-graph.ts

@@ -3,7 +3,7 @@ import { isNamedArgument } from './graph';
 import type { GenericDiffConfiguration, GenericDifferenceInformation } from '../../util/diff';
 import { setDifference } from '../../util/diff';
 import { jsonReplacer } from '../../util/json';
-import { arrayEqual } from '../../util/arrays';
+import { arrayEqual } from '../../util/collections/arrays';
 import { VertexType } from './vertex';
 import type { DataflowGraphEdge } from './edge';
 import { edgeTypesToNames, splitEdgeTypes } from './edge';

src/dataflow/graph/graph.ts

@@ -10,7 +10,7 @@ import type {
 	DataflowGraphVertices
 } from './vertex';
 import { VertexType } from './vertex';
-import { arrayEqual } from '../../util/arrays';
+import { arrayEqual } from '../../util/collections/arrays';
 import { EmptyArgument } from '../../r-bridge/lang-4.x/ast/model/nodes/r-function-call';
 import type { Identifier, IdentifierDefinition, IdentifierReference } from '../environments/identifier';
 import type { NodeId } from '../../r-bridge/lang-4.x/ast/model/processing/node-id';

src/dataflow/internal/linker.ts

@@ -1,4 +1,4 @@
-import { DefaultMap } from '../../util/defaultmap';
+import { DefaultMap } from '../../util/collections/defaultmap';
 import { guard } from '../../util/assert';
 import { expensiveTrace, log } from '../../util/log';
 import type { NodeId } from '../../r-bridge/lang-4.x/ast/model/processing/node-id';

src/dataflow/internal/process/functions/call/built-in/built-in-eval.ts

@@ -23,7 +23,7 @@ import { resolveValueOfVariable } from '../../../../../environments/resolve-by-n
 import { appendEnvironment } from '../../../../../environments/append';
 import type { RArgument } from '../../../../../../r-bridge/lang-4.x/ast/model/nodes/r-argument';
 import { isUndefined } from '../../../../../../util/assert';
-import { cartesianProduct } from '../../../../../../util/arrays';
+import { cartesianProduct } from '../../../../../../util/collections/arrays';
 
 export function processEvalCall<OtherInfo>(
 	name: RSymbol<OtherInfo & ParentInformation>,
@@ -171,4 +171,4 @@ function handlePaste(args: readonly RFunctionArgument<ParentInformation>[], env:
 	}
 
 	return result;
-}
+}

src/documentation/doc-util/doc-cfg.ts

@@ -5,7 +5,7 @@ import {
 import { requestFromInput } from '../../r-bridge/retriever';
 import type { NormalizedAst } from '../../r-bridge/lang-4.x/ast/model/processing/decorate';
 import type { KnownParser } from '../../r-bridge/parser';
-import { printAsMs } from '../../util/time';
+import { printAsMs } from '../../util/text/time';
 import { FlowrWikiBaseRef } from './doc-files';
 import type { DataflowInformation } from '../../dataflow/info';
 import { cfgToMermaid } from '../../util/mermaid/cfg';

src/documentation/doc-util/doc-dfg.ts

@@ -10,7 +10,7 @@ import { resolveDataflowGraph } from '../../dataflow/graph/resolve-graph';
 import { diffOfDataflowGraphs } from '../../dataflow/graph/diff-dataflow-graph';
 import { guard } from '../../util/assert';
 import type { PipelineOutput } from '../../core/steps/pipeline/pipeline';
-import { printAsMs } from '../../util/time';
+import { printAsMs } from '../../util/text/time';
 import type { KnownParser } from '../../r-bridge/parser';
 import { FlowrWikiBaseRef } from './doc-files';
 import { codeBlock } from './doc-code';

src/documentation/doc-util/doc-normalized-ast.ts

@@ -11,7 +11,7 @@ import { resolveDataflowGraph } from '../../dataflow/graph/resolve-graph';
 import { diffOfDataflowGraphs } from '../../dataflow/graph/diff-dataflow-graph';
 import { guard } from '../../util/assert';
 import { normalizedAstToMermaid } from '../../util/mermaid/ast';
-import { printAsMs } from '../../util/time';
+import { printAsMs } from '../../util/text/time';
 import type { KnownParser } from '../../r-bridge/parser';
 import { FlowrWikiBaseRef } from './doc-files';
 import type { GraphDifferenceReport } from '../../util/diff-graph';