feat: add configuration pipeline generator

docs/pipeline-configuration-files/getting-started.mdx

@@ -4,65 +4,37 @@ sidebar_label: "Getting Started"
 sidebar_position: 0
 ---
 
+import PipelineConfigGenerator, { PipelineIdInput } from '@site/src/components/PipelineConfigGenerator';
+
 Pipeline configuration files give you the ability to define pipelines that are
 provisioned by Conduit at startup. It's as simple as creating a YAML file that
 defines pipelines, connectors, processors, and their corresponding
 configurations.
 
-## Example pipeline
-
-:::tip
-
-In our [Conduit repository](https://github.com/ConduitIO/conduit), you can find [more examples](https://github.com/ConduitIO/conduit/tree/main/examples/pipelines), but to ilustrate a simple use case we'll show a pipeline using a file as a source, and another file as a destination. Check out the different [specifications](/docs/pipeline-configuration-files/specifications) to see the different configuration options.
-
-:::
-
-Create a folder called `pipelines` at the same level as your Conduit binary.
-Inside of that folder create a file named `file-to-file.yml`.
+When you [install](/docs/#installing) Conduit, you'll find a `pipelines` directory at the same level as your Conduit binary. This is where all your pipeline configuration files will be located **by default**:
 
 ```shell
 │ # Conduit binary
 ├── conduit
 │ # Folder with pipeline configurations
 └── pipelines
     │ # Pipeline configuration file
-    └── file-to-file.yml
+    └── source-to-destination.yml
 ```
 
-Copy following content into `file-to-file.yml`:
-
-``` yaml title="pipelines/file-to-file.yml"
-version: 2.2
-pipelines:
-  - id: file-to-file
-    # run pipeline on startup
-    status: running
-    description: >
-      Example pipeline reading from file "example.in" and writing into file "example.out".
-
-    connectors:
-      - id: example.in
-        type: source
-        # use the built-in file plugin as the source
-        plugin: builtin:file
-        settings:
-          # read data from example.in
-          path: ./example.in
-
-      - id: example.out
-        type: destination
-        # use the built-in file plugin as the destination
-        plugin: builtin:file
-        settings:
-          # write data to example.out
-          path: ./example.out
-          # output the raw payload as a string
-          sdk.record.format: template
-          sdk.record.format.options: '{{ printf "%s" .Payload.After }}'
+:::tip
+
+You can also specify a different path for the pipeline configurations directory using the `-p` flag when starting Conduit:
+
+```shell
+./conduit run -p /path/to/pipelines
 ```
 
-Now start Conduit. You should see a log line saying that the pipeline
-`file-to-file` was created:
+:::
+
+To help you get started, we've created a pipeline configuration generator that will help you create your pipeline configuration file. Take the next steps to generate your first pipeline configuration file. Check out the different [specifications](/docs/pipeline-configuration-files/specifications) to learn more about the different configuration options.
+
+<PipelineConfigGenerator />
 
 ```shell
 $ ./conduit
@@ -81,4 +53,11 @@ $ cat example.out
 hello conduit
 ```
 
+:::tip
+
+In our [Conduit repository](https://github.com/ConduitIO/conduit), you can find [more examples](https://github.com/ConduitIO/conduit/tree/main/examples/pipelines).
+
+:::
+
+
 ![scarf pixel conduit-site-docs-pipeline-config-files](https://static.scarf.sh/a.png?x-pxid=884d8c49-6ae7-41de-8868-17a3e99f28a0)

src/components/PipelineConfigGenerator/index.tsx

@@ -0,0 +1,296 @@
+import React, { useState, useEffect } from 'react';
+import yaml from 'js-yaml';
+import CodeBlock from '@theme/CodeBlock';
+import CopyButton from '@theme/CodeBlock/CopyButton';
+import useBaseUrl from '@docusaurus/useBaseUrl';
+
+// TODO: Define latest version globally somewhere in the project so we can use it everywhere
+const LATEST_VERSION = '2.2';
+export class Connector {
+  nameWithOwner: string;
+  description: string;
+  specificationPath: string;
+}
+
+export interface ConnectorItemProps {
+  connector: Connector;
+}
+
+const ConnectorItem: React.FC<ConnectorItemProps> = (props) => (
+  <option value={props.connector.description}>{props.connector.nameWithOwner}</option>
+);
+
+export const SourceConnectors = ({url, setSourceYaml, isSource}) => {
+  const [connectors, setConnectors] = useState([]);
+  const [loading, setLoading] = useState(true);
+  const [error, setError] = useState(null);
+  const [selectedValue, setSelectedValue] = useState(''); // Track selected value
+
+  useEffect(() => {
+    fetch(url)
+      .then(response => response.json())
+      .then(data => {
+        const connectors = data.filter(connector => isSource ? connector.source === "true" : connector.destination === "true");
+        setConnectors(connectors);
+        setLoading(false);
+      })
+      .catch(err => {
+        setError(err);
+        setLoading(false);
+      });
+  }, []);
+
+  const handleConnectorChange = (event) => {
+    const selectedConnector = connectors.find(connector => connector.description === event.target.value);
+    if (selectedConnector) {
+      fetch(selectedConnector.specificationPath)
+        .then(response => response.text())
+        .then(yamlData => {
+          setSourceYaml(yamlData); 
+        })
+        .catch(err => console.error('Error fetching YAML:', err));
+    }
+    setSelectedValue(event.target.value);
+  };
+
+  if (loading) {
+    return <p>Loading connectors...</p>;
+  }
+  if (error) {
+    return <p>Error: {error.message}</p>;
+  }
+
+  return (
+    <div style={{
+      backgroundColor: '#f0f0f0',
+      borderRadius: '8px',
+      padding: '15px',
+      boxShadow: '0 2px 4px rgba(0,0,0,0.1)',
+      marginBottom: '20px',
+      overflow: 'auto',
+      maxHeight: '200px',
+    }}>
+      <select 
+        value={selectedValue} 
+        onChange={handleConnectorChange}
+        style={{ width: '100%', padding: '10px', border: '1px solid #ccc', borderRadius: '4px', fontSize: '16px'}}
+      >
+        <option value="" disabled>Select a connector</option> {}
+        {connectors.map((connector) => (
+          <ConnectorItem key={connector.description} connector={connector} />
+        ))}
+      </select>
+    </div>
+  );
+}
+
+export const DestinationConnectors = ({url, setDestinationYaml, isSource}) => {
+  const [connectors, setConnectors] = useState([]);
+  const [loading, setLoading] = useState(true);
+  const [error, setError] = useState(null);
+  const [selectedValue, setSelectedValue] = useState(''); // Track selected value
+
+  useEffect(() => {
+    fetch(url)
+      .then(response => response.json())
+      .then(data => {
+        const connectors = data.filter(connector => isSource ? connector.source === "true" : connector.destination === "true");
+        setConnectors(connectors);
+        setLoading(false);
+      })
+      .catch(err => {
+        setError(err);
+        setLoading(false);
+      });
+  }, []);
+
+  const handleConnectorChange = (event) => {
+    const selectedConnector = connectors.find(connector => connector.description === event.target.value);
+    if (selectedConnector) {
+      fetch(selectedConnector.specificationPath)
+        .then(response => response.text())
+        .then(yamlData => {
+          setDestinationYaml(yamlData); 
+        })
+        .catch(err => console.error('Error fetching YAML:', err));
+    }
+    setSelectedValue(event.target.value);
+  };
+
+  if (loading) {
+    return <p>Loading connectors...</p>;
+  }
+  if (error) {
+    return <p>Error: {error.message}</p>;
+  }
+
+  return (
+    <div style={{
+      backgroundColor: '#f0f0f0',
+      borderRadius: '8px',
+      padding: '15px',
+      boxShadow: '0 2px 4px rgba(0,0,0,0.1)',
+      marginBottom: '20px',
+      overflow: 'auto',
+      maxHeight: '200px',
+    }}>
+      <select 
+        value={selectedValue} 
+        onChange={handleConnectorChange}
+        style={{ width: '100%', padding: '10px', border: '1px solid #ccc', borderRadius: '4px', fontSize: '16px'}}
+      >
+        <option value="" disabled>Select a connector</option> {}
+        {connectors.map((connector) => (
+          <ConnectorItem key={connector.description} connector={connector} />
+        ))}
+      </select>
+    </div>
+  );
+}
+
+const PipelineConfigGenerator: React.FC = () => {
+  const [sourceRawYaml, setSourceYaml] = useState('foo');
+  const [destinationRawYaml, setDestinationYaml] = useState('foo');
+  const [generatedConfig, setGeneratedConfig] = useState('');
+  const [pipelineId, setPipelineId] = useState('source-to-destination');
+
+  useEffect(() => {
+    generateConfig();
+  }, [pipelineId, sourceRawYaml, destinationRawYaml, sourceID, destinationID]);
+
+  const sourceParsedYaml = yaml.load(sourceRawYaml); 
+  const destinationParsedYaml = yaml.load(destinationRawYaml); 
+  const sourceSettings = {};
+  const destinationSettings = {};
+
+  var sourceID, destinationID, sourceName, destinationName = '';
+
+  // source
+  if (sourceParsedYaml && sourceParsedYaml.sourceParams) {
+    sourceParsedYaml.sourceParams.forEach(param => {
+      if (param.default) {
+        sourceSettings[param.name] = param.default;
+      } else {
+        sourceSettings[param.name] = '';
+      }
+    });
+  }
+
+  if (sourceParsedYaml.name) {
+    sourceID = `${sourceParsedYaml.name}-source`;
+    sourceName = `${sourceParsedYaml.name}`;
+  }
+
+  // destination
+  if (destinationParsedYaml && destinationParsedYaml.destinationParams) {
+    destinationParsedYaml.destinationParams.forEach(param => {
+      if (param.default) {
+        destinationSettings[param.name] = param.default;
+      } else {
+        destinationSettings[param.name] = '';
+      }
+    });
+  }
+
+  if (destinationParsedYaml.name) {
+    destinationID = `${destinationParsedYaml.name}-destination`;
+    destinationName = `${destinationParsedYaml.name}`;
+  }
+
+  const generateConfig = () => {
+    const config = {
+      version: LATEST_VERSION,
+      pipelines: [
+        {
+          id: pipelineId,
+          status: 'running',
+          description: `Pipeline example reading from "${sourceID}",
+and writing to "${destinationID}".`,
+          connectors: [
+            {
+              id: sourceID,
+              type: 'source',
+              // We're assuming the list of source connectors always are builtin for easier usage.
+              plugin: `builtin:${sourceName}`,
+              settings: {
+                ...sourceSettings,
+              },
+            },
+            {
+              id: destinationID,
+              type: 'destination',
+              plugin: `builtin:${destinationName}`,
+              settings: {
+                ...destinationSettings,
+              },
+            },
+          ],
+        },
+      ],
+    };
+
+    const yamlStr = yaml.dump(config);
+    setGeneratedConfig(yamlStr);
+  };
+
+  return (
+    <div>
+      <h2>1. Give your pipeline an ID</h2>
+
+      <p>Your pipeline ID is the unique identifier for your pipeline. It's used to distinguish your pipeline from other pipelines in the system. It must be unique within your pipeline configuration file.</p>
+
+      <div style={{
+        backgroundColor: '#f0f0f0',
+        borderRadius: '8px',
+        padding: '15px',
+        boxShadow: '0 2px 4px rgba(0,0,0,0.1)',
+        marginBottom: '20px'
+      }}>
+        <input
+          id="pipelineId"
+          type="text"
+          value={pipelineId}
+          onChange={(e) => setPipelineId(e.target.value)}
+          style={{
+            width: '100%',
+            padding: '10px',
+            boxSizing: 'border-box',
+            fontSize: '16px',
+            border: '1px solid #ccc',
+            borderRadius: '5px',
+            backgroundColor: 'white'
+          }}
+          placeholder="e.g., my-first-pipeline"
+        />
+      </div>
+
+      <h2>2. Choose your source connector</h2>
+
+      <SourceConnectors url={useBaseUrl('/connectors.json')} setSourceYaml={setSourceYaml} isSource={true}/>
+
+      <h2>3. Choose your destination connector</h2>
+
+      <DestinationConnectors url={useBaseUrl('/connectors.json')} setDestinationYaml={setDestinationYaml} isSource={false}/>
+
+      <h2>4. Copy the generated pipeline configuration file</h2>
+
+      {generatedConfig && (
+        <div>
+          <div style={{ position: 'relative' }}>
+          <CodeBlock
+            language="yaml"
+            showLineNumbers>
+            <div className="buttonGroup_node_modules-@docusaurus-theme-classic-lib-theme-CodeBlock-Content-styles-module" >
+              <CopyButton code={generatedConfig} />
+            </div>
+            {generatedConfig} 
+          </CodeBlock>
+          </div>
+        </div>
+      )}
+      <p>Now, you need to include this into your previously created file and start Conduit. You should seeing log lines saying that the pipeline <code>{pipelineId}</code> was created and ready to stream process:</p>
+    </div>
+  );
+};
+
+export default PipelineConfigGenerator;