Skip to content

Commit 40b2ba6

Browse files
melden19melden19
committed
Add UI pipeline runner
1 parent 1fd861e commit 40b2ba6

34 files changed

+3639
-1294
lines changed

‎compositions/docker-compose.yml

+6
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,16 @@ services:
33
server:
44
build:
55
context: ../packages/server
6+
environment:
7+
CONTAINER_TIMEOUT: 10000 # 10 sec
8+
69
depends_on:
710
- mongo
811
ports:
912
- 8080:8080
13+
volumes:
14+
- /usr/bin/docker:/usr/bin/docker
15+
- /var/run/docker.sock:/var/run/docker.sock
1016
ui:
1117
build:
1218
context: ../packages/ui

‎index.js

-3
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,5 @@ dockerdListener.start(container => {
1818
// storage.init().then(() => {
1919
// storage.read().then(res => {
2020
// console.log(res);
21-
// const a = process._getActiveHandles();
22-
// const b = process._getActiveRequests();
23-
// console.log(a, b);
2421
// });
2522
// });

‎package.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,8 @@
77
"test": "echo \"Error: no test specified\" && exit 1",
88
"engine": "node src/engine",
99
"start:prod": "docker-compose -f compositions/docker-compose.yml -f compositions/docker-compose.prod.yml up --build",
10-
"start:dev": "docker-compose -f compositions/docker-compose.yml -f compositions/docker-compose.dev.yml up --build"
10+
"start:dev": "docker-compose -f compositions/docker-compose.yml -f compositions/docker-compose.dev.yml up --build",
11+
"start:server": "docker-compose -f compositions/docker-compose.yml -f compositions/docker-compose.dev.yml up --build server"
1112
},
1213
"keywords": [],
1314
"author": "",

‎packages/server/config.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,10 @@ const mongo = {};
1717
if (process.env.NODE_ENV === 'prod') {
1818
mongo.uri = process.env.MONGO_URI || 'mongodb://localhost:27017/docker-logger';
1919
} else {
20-
mongo.uri = `mongodb://${debug ? 'localhost' : 'mongo'}/docker-logger`
20+
mongo.uri = `mongodb://${debug ? 'localhost:27017' : 'mongo'}/docker-logger`
2121
}
2222

23-
const containerTimeout = 1000 * 60 * process.env.CONTAINER_TIMEOUT || 10; // default 10 min
23+
const containerTimeout = process.env.CONTAINER_TIMEOUT || 20000; // default 20 sec
2424
const availableStorageLayers = ['fs', 'mongo'];
2525

2626
module.exports = {

‎packages/server/src/api/containers/index.js

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
const express = require('express');
2+
const { checkStorageLayer } = require('../../middlewares');
3+
const createStorageLayer = require('../../storageLayers');
4+
5+
const router = express.Router();
6+
7+
router.get('/:containerId/logs', checkStorageLayer, async (req, res, next) => {
8+
try {
9+
const storage = createStorageLayer(req.query.storageLayer, 'reader', { id: req.params.containerId });
10+
const data = await storage.read();
11+
res.json(data);
12+
} catch(err) {
13+
next(err);
14+
}
15+
});
16+
17+
module.exports = router;

‎packages/server/src/api/index.js

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
const express = require('express');
2+
const router = express.Router();
3+
4+
router.use('/containers', require('./containers'));
5+
router.use('/pipelines', require('./pipelines'));
6+
7+
module.exports = router;

‎packages/server/src/api/pipelines/index.js

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
const express = require('express');
2+
const Engine = require('../../engine');
3+
const { validatePipeline, checkStorageLayer } = require('../../middlewares');
4+
5+
const router = express.Router();
6+
7+
router.get('/', async (req, res) => {
8+
res.json({ foo: 'bar' });
9+
});
10+
11+
router.post('/run', validatePipeline, checkStorageLayer, async ({ body }, res) => {
12+
const engine = new Engine({ pipeline: body.pipeline, storageLayer: body.storageLayer });
13+
const result = await engine.run();
14+
res.json(result);
15+
});
16+
17+
module.exports = router;

‎packages/server/src/app.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ const mongoose = require('mongoose');
66
const chalk = require('chalk');
77

88
const { mongo } = require('../config');
9-
const index = require('./routes/index');
9+
const index = require('./api/index');
1010

1111
const app = express();
1212
const server = http.createServer(app);

‎packages/server/src/engine/ContainerLogger/index.js

+2-4
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
const { promisify } = require('util');
2-
const eventDebug = require('event-debug');
3-
const { logContainer, delay } = require('../../helpers');
2+
const { logContainer } = require('../../helpers');
43
const { docker } = require('../../../config');
54

65
class ContainerLogger {
@@ -12,6 +11,7 @@ class ContainerLogger {
1211

1312
async run() {
1413
try {
14+
await this._initStorageLayer();
1515
await this._exec();
1616
} catch(err) {
1717
console.error('Failed to run ContainerLogger', err);
@@ -24,8 +24,6 @@ class ContainerLogger {
2424

2525
async _exec() {
2626
const stream = await this._attachToContainer();
27-
await this._initStorageLayer();
28-
eventDebug(stream);
2927

3028
stream.on('end', () => {
3129
this.storage.stream.end();

‎packages/server/src/engine/PipelineRunner/index.js

+57-14
Original file line numberDiff line numberDiff line change
@@ -1,58 +1,99 @@
11
const { readFileSync } = require('fs');
2+
const { execSync } = require('child_process');
23
const _ = require('lodash');
34
const yaml = require('js-yaml');
4-
const { docker, storageLayer } = require('../../../config');
5+
const { docker, containerTimeout } = require('../../../config');
56
const startListener = require('../StartListener');
67
const createStorageLayer = require('../../storageLayers');
7-
const { logContainer, normalizeId } = require('../../helpers');
8+
const { normalizeId, logContainer } = require('../../helpers');
89
const Logger = require('../ContainerLogger');
910

11+
const STEP_LABEL = 'codefresh-assessment-pipeline-step';
12+
1013
class PipelineRunner {
1114
constructor(settings = {}) {
12-
const { path } = settings;
15+
const { path, pipeline, storageLayer } = settings;
1316
this.path = path || './pipeline.yml';
17+
this.pipeline = pipeline;
18+
this.storageLayer = storageLayer;
1419
}
1520

1621
async run() {
22+
const executionResult = [];
23+
1724
try {
1825
const pipeline = this._loadPipeline();
1926
this._validatePipeline(pipeline);
2027

2128
await this._initListener();
2229

2330
for (const [key, value] of Object.entries(pipeline.steps)) {
24-
const { StatusCode } = await this._runStep({ name: key, ...value });
25-
if (StatusCode > 0) {
26-
throw new Error(`Pipeline failed on step ${key} with status code ${StatusCode}`);
31+
const container = await this._runStep({ name: key, ...value });
32+
normalizeId(container);
33+
executionResult.push({ name: key, ...container });
34+
35+
if (container.exitCode > 0) {
36+
if (container.exitCode === 143) {
37+
throw new Error(`Container was stopped cause of predefined max execution timeout: ${containerTimeout / 1000} sec`);
38+
}
39+
throw new Error(`Pipeline failed on step ${key} with status code ${container.statusCode}`);
2740
}
2841
}
2942
console.log('Pipeline run successfully');
30-
this._killListener();
43+
this._gracefulShutdown();
44+
return { executionResult, status: 'Pipeline run successfully' };
3145

3246
} catch (err) {
3347
console.error('Failed to run pipeline cause of: ', err);
34-
this._killListener();
48+
this._gracefulShutdown();
49+
return { executionResult, status: `Failed to run pipeline cause of: ${err}`, error: true };
3550
}
3651
}
3752

3853
async _runStep(step) {
39-
const { image: Image, commands: Cmd, name } = step;
54+
const { image: Image, cmd: Cmd, name } = step;
4055

4156
try {
42-
const container = await docker.createContainer({ name, Image, Cmd, AttachStdout: true, AttachStderr: true });
57+
const container = await docker.createContainer({
58+
name,
59+
Image,
60+
Cmd,
61+
Labels: {
62+
meta: STEP_LABEL
63+
}
64+
});
4365
const output = await container.start();
66+
let running = true;
67+
4468
if (output.Error) {
4569
throw output.Error;
4670
}
71+
72+
setTimeout(() => {
73+
if (running) {
74+
container.stop().catch((err) => {
75+
console.error('Container timeout termination failed', err);
76+
});
77+
}
78+
}, containerTimeout);
79+
4780
const res = await container.wait();
48-
return res;
81+
running = false;
82+
83+
return {
84+
exitCode: res.StatusCode,
85+
id: container.id
86+
};
4987
} catch (err) {
5088
throw new Error(`Failed to run step cause of: ${err}`);
5189
}
5290
}
5391

5492
_loadPipeline() {
5593
try {
94+
if (this.pipeline) {
95+
return this.pipeline;
96+
}
5697
const file = readFileSync(this.path);
5798
return yaml.safeLoad(file);
5899
} catch (err) {
@@ -67,17 +108,19 @@ class PipelineRunner {
67108
}
68109

69110
_initListener() {
70-
return startListener.start(async (container) => {
111+
return startListener.start({ labels: [`meta=${STEP_LABEL}`] }, async (container) => {
71112
normalizeId(container);
113+
logContainer('Attaching to container', container);
72114

73-
const storage = createStorageLayer(storageLayer, 'writer', container);
115+
const storage = createStorageLayer(this.storageLayer, 'writer', container);
74116

75117
const logger = new Logger(container.id, storage);
76118
await logger.run();
77119
});
78120
}
79121

80-
_killListener() {
122+
_gracefulShutdown() {
123+
execSync('yes | docker container prune');
81124
startListener.stop();
82125
}
83126
}

‎packages/server/src/engine/StartListener/index.js

+10-7
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
const _ = require('lodash');
2-
const { selector, docker } = require('../../../config');
2+
const { docker } = require('../../../config');
33

44
class StartListener {
5-
start(containerHandler) {
5+
start(selector, containerHandler) {
66
return new Promise((resolve, reject) => {
7-
const filters = this._getApiFilter();
7+
const filters = this._getApiFilter(selector);
88
docker.getEvents({ filters }, (err, events) => {
99
if (err) {
1010
reject(err);
@@ -13,7 +13,7 @@ class StartListener {
1313
this.events = events;
1414
events.on('data', (chunk) => {
1515
const containerData = JSON.parse(chunk.toString());
16-
if (this._matchLocalFilters(containerData)) {
16+
if (this._matchLocalFilters(containerData, selector)) {
1717
containerHandler(containerData)
1818
}
1919
});
@@ -35,18 +35,21 @@ class StartListener {
3535

3636

3737

38-
_matchLocalFilters(eventData) {
38+
_matchLocalFilters(eventData, selector) {
3939
const name = _.get(eventData, 'Actor.Attributes.name');
4040
if (selector.nameRegex && name) {
4141
return (new RegExp(selector.nameRegex)).test(name);
4242
}
4343
return true;
4444
}
4545

46-
_getApiFilter() {
46+
_getApiFilter(selector) {
4747
const filter = {};
4848
_.set(filter, 'event', ['start']);
49-
_.set(filter, 'label', selector.labels);
49+
50+
if (selector.labels) {
51+
_.set(filter, 'label', _.castArray(selector.labels));
52+
}
5053

5154
return JSON.stringify(filter);
5255
}

‎packages/server/src/middlewares.js

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
const { availableStorageLayers } = require('../config');
2+
const yaml = require('js-yaml');
3+
4+
module.exports.checkStorageLayer = (req, res, next) => {
5+
const storageLayerHolder = req.method === 'GET' ? 'query' : 'body';
6+
7+
const { storageLayer } = req[storageLayerHolder];
8+
if (!storageLayer) {
9+
req[storageLayerHolder].storageLayer = 'mongo';
10+
}
11+
if (!availableStorageLayers.includes(storageLayer)) {
12+
next(new Error('Storage layer is not available'));
13+
}
14+
next();
15+
16+
};
17+
18+
module.exports.validatePipeline = (req, res, next) => {
19+
if (req.body.pipeline) {
20+
const decodedPipeline = Buffer.from(req.body.pipeline, 'base64').toString();
21+
try {
22+
req.body.pipeline = yaml.safeLoad(decodedPipeline);
23+
next();
24+
} catch(err) {
25+
next(err);
26+
}
27+
} else {
28+
next(new Error('Please, provide pipeline'));
29+
}
30+
};

‎packages/server/src/routes/index.js

-28
This file was deleted.

‎packages/server/src/storageLayers/Mongo/Mongo.reader.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,12 @@ const mongoose = require('mongoose');
22

33
const BaseStorageLayer = require('../Base');
44
const ContainerLog = require('./Log');
5-
const { mongo, debug } = require('../../../config');
5+
const { mongo } = require('../../../config');
66
const { notFoundMessage } = require('../constants');
77

88
class MongoStorageRead extends BaseStorageLayer.Reader {
99
async init() {
10-
if (debug) {
10+
if (mongoose.connection.readyState === 0) {
1111
return this._initMongo();
1212
}
1313
}

0 commit comments

Comments
 (0)