diff --git a/.eslintrc.yml b/.eslintrc.yml index b3d07387..b80a5c8f 100644 --- a/.eslintrc.yml +++ b/.eslintrc.yml @@ -39,9 +39,6 @@ parserOptions: templateStrings: true unicodeCodePointEscapes: true rules: - arrow-parens: - - 2 - - as-needed array-bracket-spacing: - 2 - never @@ -58,7 +55,6 @@ rules: comma-style: - 2 - last - complexity: [1, 9] computed-property-spacing: - 2 - never @@ -290,4 +286,4 @@ rules: newline-per-chained-call: 0 class-methods-use-this: 0 no-empty-function: 0 - sort-imports: [2, { memberSyntaxSortOrder: ['none', 'all', 'multiple', 'single'], ignoreCase: true }] + sort-imports: 0 diff --git a/CHANGELOG.md b/CHANGELOG.md index c9c01b30..4f42b73c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -514,7 +514,7 @@ Big thanks to @shaiamir for his work on the shared proxy. - Change mongo version - Validates `mup.js` and displays problems found in it - Update message is clearer and more colorful -- `uploadProgressBar` is part of default `mup.js` +- `enableUploadProgressBar` is part of default `mup.js` - Add trailing commas to mup.js (@ffxsam) - Improve message when settings.json is not found or is invalid - Loads and parses settings.json before building the app diff --git a/docs/docs.md b/docs/docs.md index 0bc98ce8..eae68e7b 100644 --- a/docs/docs.md +++ b/docs/docs.md @@ -139,7 +139,7 @@ module.exports = { // make deploys more reliable and easier to troubleshoot prepareBundle: true, - // (optional, default is false) Uses the new docker image builder + // (optional, default is true) Uses the new docker image builder // during Prepare bundle. When enabled, // Prepare Bundle is much faster useBuildKit: true, @@ -236,12 +236,7 @@ module.exports = { // lets you define which port to check after the deploy process, if it // differs from the meteor port you are serving // (like meteor behind a proxy/firewall) (optional) - deployCheckPort: 80, - - // Shows progress bar while uploading bundle to server - // You might need to disable it on CI servers - // (optional, default is false) - enableUploadProgressBar: true + deployCheckPort: 80 }, // (optional) Use built-in mongodb. Remove it to use a remote MongoDB diff --git a/package-lock.json b/package-lock.json index 2db0e9aa..6ffc61f5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "mup", - "version": "1.5.8", + "version": "1.6.0-beta.3", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "mup", - "version": "1.5.8", + "version": "1.6.0-beta.3", "hasInstallScript": true, "license": "MIT", "dependencies": { diff --git a/package.json b/package.json index 53b1b270..6edd9f49 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "mup", - "version": "1.5.9", + "version": "1.6.0-beta.5", "description": "Production Quality Meteor Deployments", "main": "lib/index.js", "repository": { diff --git a/src/__tests__/plugin-api.unit.js b/src/__tests__/plugin-api.unit.js index 5234a3c1..187660a5 100644 --- a/src/__tests__/plugin-api.unit.js +++ b/src/__tests__/plugin-api.unit.js @@ -169,7 +169,7 @@ describe('PluginAPI', () => { describe('_normalizeConfig', () => { it('should copy meteor object to app', () => { - const expected = { meteor: { path: '../' }, app: { type: 'meteor', path: '../', docker: { image: 'kadirahq/meteord', imagePort: 3000, stopAppDuringPrepareBundle: true } } }; + const expected = { meteor: { path: '../' }, app: { type: 'meteor', path: '../', docker: { image: 'zodern/meteor:0.6.1-root', imagePort: 3000, stopAppDuringPrepareBundle: true } } }; const config = { meteor: { path: '../' } }; const result = api._normalizeConfig(config); diff --git a/src/check-setup.js b/src/check-setup.js new file mode 100644 index 00000000..c5896cd8 --- /dev/null +++ b/src/check-setup.js @@ -0,0 +1,89 @@ +/* eslint-disable no-labels */ +const crypto = require('crypto'); +const fs = require('fs'); + +const checkers = []; + +export function registerChecker(checker) { + checkers.push(checker); +} + +function createKey(keyConfig = {}) { + const finalConfig = { + ...keyConfig + }; + if (finalConfig.scripts) { + finalConfig.scripts = finalConfig.scripts.map( + path => fs.readFileSync(path, 'utf-8') + ); + } + + return crypto.createHash('sha256') + .update(JSON.stringify(finalConfig)) + .digest('base64'); +} + +export async function checkSetup(pluginApi) { + const checks = await Promise.all(checkers.map(checker => checker(pluginApi))); + + console.time('setup check'); + const bySession = new Map(); + checks.flat().forEach(check => { + let keyHash = createKey(check.setupKey); + + check.sessions.forEach(session => { + const config = bySession.get(session) || { + keyHashes: {}, + services: [], + containers: [] + }; + + config.keyHashes[check.name] = keyHash; + config.services.push(...check.services || []); + config.containers.push(...check.containers || []); + bySession.set(session, config); + }); + }); + + const promises = []; + + bySession.forEach((config, session) => { + const promise = new Promise(resolve => { + session.executeScript( + pluginApi.resolvePath(__dirname, './tasks/assets/check-setup.sh'), + { + vars: config + }, + (err, code) => { + resolve(!err && code === 0); + } + ); + }); + promises.push(promise); + }); + + const result = await Promise.all(promises); + console.timeEnd('setup check'); + + + return result.every(upToDate => upToDate); +} + +export async function updateSetupKeys(api) { + const checks = await Promise.all(checkers.map(checker => checker(api))); + + // TODO: parallelize this + for (const check of checks.flat()) { + const setupKeyHash = createKey(check.setupKey); + + for (const session of check.sessions) { + // TODO: handle errors. Should retry, and after 3 tries give up + // Shouldn't throw since if the command fails mup will simply setup + // again next time + await api.runSSHCommand( + session, + `sudo mkdir -p /opt/.mup-setup && sudo echo "${setupKeyHash}" > /opt/.mup-setup/${check.name}.txt` + ); + } + } +} diff --git a/src/index.js b/src/index.js index bd8f3d2b..f119b174 100644 --- a/src/index.js +++ b/src/index.js @@ -40,7 +40,7 @@ if (config.hooks) { } function commandWrapper(pluginName, commandName) { - return function() { + return async function() { // Runs in parallel with command checkUpdates([ { name: pkg.name, path: require.resolve('../package.json') }, @@ -52,6 +52,8 @@ function commandWrapper(pluginName, commandName) { const api = new MupAPI(process.cwd(), filteredArgv, yargs.argv); let potentialPromise; + await api.loadServerGroups(); + try { potentialPromise = api.runCommand(`${pluginName}.${commandName}`); } catch (e) { diff --git a/src/load-plugins.js b/src/load-plugins.js index babb1616..c265757e 100644 --- a/src/load-plugins.js +++ b/src/load-plugins.js @@ -9,7 +9,9 @@ import registerCommand from './commands'; import { registerHook } from './hooks'; import { registerPreparer } from './prepare-config'; import { registerScrubber } from './scrub-config'; +import { registerServerSource } from './server-sources'; import { registerSwarmOptions } from './swarm-options'; +import { registerChecker } from './check-setup'; import resolveFrom from 'resolve-from'; const log = debug('mup:plugin-loader'); @@ -72,7 +74,14 @@ export function locatePluginDir(name, configPath, appPath) { function registerPlugin(plugin) { if (plugin.module.commands) { Object.keys(plugin.module.commands).forEach(key => { - registerCommand(plugin.name, key, plugin.module.commands[key]); + let command = plugin.module.commands[key]; + registerCommand( + // The __plugin option can be used to change the top-level command + // the command is added to + command.__plugin || plugin.name, + key, + plugin.module.commands[key] + ); }); } if (plugin.module.hooks) { @@ -95,6 +104,14 @@ function registerPlugin(plugin) { if (plugin.module.swarmOptions) { registerSwarmOptions(plugin.module.swarmOptions); } + if (plugin.module.serverSources) { + for (const [type, config] of Object.entries(plugin.module.serverSources)) { + registerServerSource(type, config); + } + } + if (plugin.module.checkSetup) { + registerChecker(plugin.module.checkSetup); + } } export function loadPlugins(plugins) { diff --git a/src/nodemiral.js b/src/nodemiral.js index 13d71010..1f795f90 100644 --- a/src/nodemiral.js +++ b/src/nodemiral.js @@ -63,13 +63,18 @@ function createCallback(cb, varsMapper) { return cb(err); } if (code > 0) { - const message = ` - ------------------------------------STDERR------------------------------------ - ${logs.stderr.substring(logs.stderr.length - 4200)} - ------------------------------------STDOUT------------------------------------ - ${logs.stdout.substring(logs.stdout.length - 4200)} - ------------------------------------------------------------------------------ - `; + let message = ''; + if (!logs.stderr.length && logs.stdout.length) { + message = logs.stdout.substring(logs.stdout.length - 8400); + } else { + message = ` + ------------------------------------STDERR------------------------------------ + ${logs.stderr.substring(logs.stderr.length - 4200)} + ------------------------------------STDOUT------------------------------------ + ${logs.stdout.substring(logs.stdout.length - 4200)} + ------------------------------------------------------------------------------ + `; + } return cb(new Error(message)); } @@ -82,8 +87,24 @@ function createCallback(cb, varsMapper) { }; } +// TODO: running hooks should be an option for executeScript instead of +// a separate task +async function runDuringHooks(session, options, callback) { + console.dir(options); + const pluginApi = options._getMupApi(); + // console.dir(pluginApi); + try { + await pluginApi._runDuringHooks(options.hookName, session); + } catch (e) { + return callback(e); + } + + callback(); +} + nodemiral.registerTask('copy', copy); nodemiral.registerTask('executeScript', executeScript); +nodemiral.registerTask('_runHook', runDuringHooks); const oldApplyTemplate = nodemiral.session.prototype._applyTemplate; // Adds support for using include with ejs diff --git a/src/plugin-api.js b/src/plugin-api.js index b1cc239a..05b2b2a9 100644 --- a/src/plugin-api.js +++ b/src/plugin-api.js @@ -17,6 +17,8 @@ import path from 'path'; import { runConfigPreps } from './prepare-config'; import { scrubConfig } from './scrub-config'; import serverInfo from './server-info'; +import { serverSources } from './server-sources'; +import { checkSetup, updateSetupKeys } from './check-setup'; const { resolvePath, moduleNotFoundIsPath } = utils; const log = debug('mup:api'); @@ -29,6 +31,7 @@ export default class PluginAPI { this.config = null; this.settings = null; this.sessions = null; + this._serverGroupServers = Object.create(null); this._enabledSessions = program.servers ? program.servers.split(' ') : []; this.configPath = program.config ? resolvePath(program.config) : path.join(this.base, 'mup.js'); this.settingsPath = program.settings; @@ -95,6 +98,8 @@ export default class PluginAPI { opts.showDuration = this.profileTasks; } + opts._mupPluginApi = this; + return utils.runTaskList(list, sessions, opts); } @@ -127,9 +132,9 @@ export default class PluginAPI { ); console.log(' http://meteor-up.com/docs'); console.log(''); - } - this.validationErrors = problems; + this.validationErrors = problems; + } return problems; } @@ -244,7 +249,7 @@ export default class PluginAPI { process.exit(1); } } - _runHooks = async function(handlers, hookName) { + _runHooks = async function(handlers, hookName, secondArg) { const messagePrefix = `> Running hook ${hookName}`; for (const hookHandler of handlers) { @@ -254,7 +259,7 @@ export default class PluginAPI { } if (typeof hookHandler.method === 'function') { try { - await hookHandler.method(this, nodemiral); + await hookHandler.method(this, secondArg || nodemiral); } catch (e) { this._commandErrorHandler(e); } @@ -270,6 +275,19 @@ export default class PluginAPI { } } } + async _runDuringHooks(name, session) { + const hookName = `during.${name}`; + + if (this.program['show-hook-names']) { + console.log(chalk.yellow(`Hook: ${hookName}`)); + } + + if (hookName in hooks) { + const hookList = hooks[hookName]; + + await this._runHooks(hookList, name, { session }); + } + } _runPreHooks = async function(name) { const hookName = `pre.${name}`; @@ -335,24 +353,49 @@ export default class PluginAPI { this._commandErrorHandler(e); } - await this._runPostHooks(name).then(() => { - // The post hooks for the first command should be the last thing run - if (firstCommand) { - this._cleanupSessions(); + await this._runPostHooks(name); + + if (name === 'default.setup') { + console.log('=> Storing setup config on servers'); + await updateSetupKeys(this); + } + + // The post hooks for the first command should be the last thing run + if (firstCommand) { + this._cleanupSessions(); + } + } + + expandServers(serversObj) { + let result = {}; + const serverConfig = this.getConfig().servers; + + Object.entries(serversObj).forEach(([key, config]) => { + if (key in this._serverGroupServers) { + this._serverGroupServers[key].forEach(server => { + result[server.name] = { server, config }; + }); + } else { + result[key] = { + server: serverConfig[key], + config + }; } }); + + return result; } async getServerInfo(selectedServers, collectors) { if (this._cachedServerInfo && !collectors) { return this._cachedServerInfo; } - const serverConfig = this.getConfig().servers; + const serverConfig = this.expandServers(this.getConfig().servers); const servers = ( - selectedServers || Object.keys(this.getConfig().servers) + selectedServers || Object.keys(serverConfig) ).map(serverName => ({ - ...serverConfig[serverName], + ...serverConfig[serverName].server, name: serverName })); @@ -374,6 +417,80 @@ export default class PluginAPI { this._cachedServerInfo = null; } + async checkSetupNeeded() { + const [upToDate, serverGroupsUpToDate] = await Promise.all([ + checkSetup(this), + this._serverGroupsUpToDate() + ]); + + return !upToDate || !serverGroupsUpToDate; + } + + _mapServerGroup(cb) { + const { servers } = this.getConfig(false); + + if (typeof servers !== 'object' || servers === null) { + return []; + } + + return Object.entries(servers) + .filter(([, serverConfig]) => serverConfig && typeof serverConfig.source === 'string') + .map(async ([name, serverConfig]) => { + const source = serverConfig.source; + + if (!(source in serverSources)) { + throw new Error(`Unrecognized server source: ${source}. Available: ${Object.keys(serverSources)}`); + } + + return cb(name, serverConfig); + }); + } + + async loadServerGroups() { + const promises = this._mapServerGroup(async (name, groupConfig) => { + const source = groupConfig.source; + const list = await serverSources[source].load( + { name, groupConfig }, this + ); + this._serverGroupServers[name] = list; + + // TODO: handle errors. We should delay throwing the error until + // we need the sessions from this server group + }); + + await Promise.all(promises); + } + + async _serverGroupsUpToDate() { + const promises = this._mapServerGroup((name, groupConfig) => { + const source = groupConfig.source; + + return serverSources[source].upToDate({ name, groupConfig }, this); + }); + + const result = await Promise.all(promises); + + return result.every(upToDate => upToDate); + } + + async updateServerGroups() { + this.sessions = null; + + const promises = this._mapServerGroup(async (name, groupConfig) => { + const source = groupConfig.source; + await serverSources[source].update({ name, groupConfig }, this); + const list = await serverSources[source].load( + { name, groupConfig }, this + ); + this._serverGroupServers[name] = list; + }); + + await Promise.all(promises).catch(e => { + console.dir(e); + throw e; + }); + } + getSessions(modules = []) { const sessions = this._pickSessions(modules); @@ -385,7 +502,20 @@ export default class PluginAPI { this._loadSessions(); } - return servers.map(name => this.sessions[name]); + let result = []; + + servers.forEach(name => { + let session = this.sessions[name]; + if (Array.isArray(session)) { + session.forEach(memberName => { + result.push(this.sessions[memberName]); + }); + } else { + result.push(session); + } + }); + + return result; } async getManagerSession() { @@ -413,7 +543,16 @@ export default class PluginAPI { continue; } - if (this.sessions[name]) { + if (!this.sessions[name]) { + continue; + } + + if (Array.isArray(this.sessions[name])) { + // Is a server group. Add the members of the group. + this.sessions[name].forEach(memberName => { + sessions[memberName] = this.sessions[memberName]; + }); + } else { sessions[name] = this.sessions[name]; } } @@ -427,21 +566,7 @@ export default class PluginAPI { this.sessions = {}; - // `mup.servers` contains login information for servers - // Use this information to create nodemiral sessions. - for (const name in config.servers) { - if (!config.servers.hasOwnProperty(name)) { - continue; - } - - if ( - this._enabledSessions.length > 0 && - this._enabledSessions.indexOf(name) === -1 - ) { - continue; - } - - const info = config.servers[name]; + function createNodemiralSession(name, info) { const auth = { username: info.username }; @@ -478,9 +603,38 @@ export default class PluginAPI { process.exit(1); } - const session = nodemiral.session(info.host, auth, opts); + return nodemiral.session(info.host, auth, opts); + } + + // `mup.servers` contains login information for servers + // Use this information to create nodemiral sessions. + for (const name in config.servers) { + if (!config.servers.hasOwnProperty(name)) { + continue; + } - this.sessions[name] = session; + if ( + this._enabledSessions.length > 0 && + this._enabledSessions.indexOf(name) === -1 + ) { + // TODO: if server group, check if any servers in group are enabled + continue; + } + + const info = config.servers[name]; + + if (typeof info.source === 'string') { + const servers = this._serverGroupServers[name]; + servers.forEach(server => { + const session = createNodemiralSession( + server.name, server + ); + this.sessions[server.name] = session; + }); + this.sessions[name] = servers.map(s => s.name); + } else { + this.sessions[name] = createNodemiralSession(name, info); + } } } @@ -491,7 +645,9 @@ export default class PluginAPI { } Object.keys(this.sessions).forEach(key => { - this.sessions[key].close(); + if (!Array.isArray(this.sessions[key])) { + this.sessions[key].close(); + } }); } diff --git a/src/plugins/default/command-handlers.js b/src/plugins/default/command-handlers.js index 1284389b..2d2ae9e3 100644 --- a/src/plugins/default/command-handlers.js +++ b/src/plugins/default/command-handlers.js @@ -5,23 +5,40 @@ import { map } from 'bluebird'; const log = debug('mup:module:default'); -export function deploy() { +export async function deploy(api) { log('exec => mup deploy'); + console.log('=> Checking if server setup needed'); + if (await api.checkSetupNeeded()) { + await api.runCommand('default.setup'); + } } export function logs() { log('exec => mup logs'); } -export function reconfig() { +export async function reconfig(api) { log('exec => mup reconfig'); + + if (api.commandHistory.find(({ name }) => name === 'default.deploy')) { + // We've already checked if setup is needed + return; + } + + console.log('=> Checking if server setup needed'); + if (await api.checkSetupNeeded()) { + await api.runCommand('default.setup'); + } } export function restart() { log('exec => mup restart'); } -export function setup(api) { +export async function setup(api) { + log('exec => mup setup'); + await api.updateServerGroups(); + process.on('exit', code => { if (code > 0) { return; @@ -32,7 +49,6 @@ export function setup(api) { console.log(' mup deploy'); }); - log('exec => mup setup'); return api.runCommand('docker.setup'); } @@ -47,6 +63,7 @@ export function stop() { export function ssh(api) { const servers = api.getConfig().servers; + const expandedServers = api.expandServers(servers); let serverOption = api.getArgs()[1]; // Check how many sessions are enabled. Usually is all servers, @@ -54,7 +71,7 @@ export function ssh(api) { const enabledSessions = api.getSessionsForServers(Object.keys(servers)) .filter(session => session); - if (!(serverOption in servers)) { + if (!(serverOption in expandedServers)) { if (enabledSessions.length === 1) { const selectedHost = enabledSessions[0]._host; serverOption = Object.keys(servers).find( @@ -62,14 +79,14 @@ export function ssh(api) { ); } else { console.log('mup ssh '); - console.log('Available servers are:\n', Object.keys(servers).join('\n ')); + console.log('Available servers are:\n', Object.keys(expandedServers).join('\n ')); process.exitCode = 1; return; } } - const server = servers[serverOption]; + const server = expandedServers[serverOption].server; const sshOptions = api._createSSHOptions(server); const conn = new Client(); @@ -151,12 +168,14 @@ function statusColor( } export async function status(api) { - const servers = Object.values(api.getConfig().servers || {}); + const config = api.getConfig(); + const allServers = Object.values(api.expandServers(config.servers || {})) + .map(({ server }) => server); const lines = []; let overallColor = 'green'; const command = 'lsb_release -r -s || echo "false"; lsb_release -is; apt-get -v &> /dev/null && echo "true" || echo "false"; echo $BASH'; const results = await map( - servers, + allServers, server => api.runSSHCommand(server, command), { concurrency: 2 } ); @@ -199,6 +218,12 @@ export async function status(api) { lines.push(text); }); + if (lines.length === 0) { + overallColor = 'gray'; + lines.push(chalk.gray(' No servers listed in config')); + } + console.log(chalk[overallColor]('=> Servers')); console.log(lines.join('\n')); + console.log(''); } diff --git a/src/plugins/default/index.js b/src/plugins/default/index.js index afb30694..aeaaa0d5 100644 --- a/src/plugins/default/index.js +++ b/src/plugins/default/index.js @@ -1,8 +1,11 @@ import * as _commands from './commands'; +import _serverSources from './server-groups/index.js'; import traverse from 'traverse'; export const commands = _commands; +export const serverSources = _serverSources; + export function scrubConfig(config) { if (config.servers) { // eslint-disable-next-line diff --git a/src/plugins/default/init.js b/src/plugins/default/init.js index fa6c8ec4..aa531ff9 100644 --- a/src/plugins/default/init.js +++ b/src/plugins/default/init.js @@ -99,8 +99,10 @@ export default function init(api) { console.log(' Available options can be found in the docs at'); console.log(' https://github.com/zodern/meteor-up'); console.log(''); - console.log(' Then, run the command:'); + console.log(' Once the config is ready, you can setup your servers with:'); console.log(' mup setup'); + console.log(' And deploy your app by running:'); + console.log(' mup deploy'); } else { console.log('Skipping creation of mup.js'); console.log(`mup.js already exists at ${configDest}`); diff --git a/src/plugins/default/server-groups/digital-ocean.js b/src/plugins/default/server-groups/digital-ocean.js new file mode 100644 index 00000000..93c8d83d --- /dev/null +++ b/src/plugins/default/server-groups/digital-ocean.js @@ -0,0 +1,275 @@ +import axios from 'axios'; +import { generateName, createFingerprint } from './utils'; +import fs from 'fs'; + +export default class DigitalOcean { + constructor(groupName, groupConfig, pluginApi) { + this.name = groupName; + this.config = groupConfig; + + this.publicKeyPath = pluginApi.resolvePath(this.config.sshKey.public); + + this.tag = groupConfig.__tag || `mup-${this.name}`; + } + + async getServers(ids) { + // TODO: implement pagination + const results = await this._request( + 'get', + `droplets?tag_name=${this.tag}&per_page=200` + ); + + return results.data.droplets.map(droplet => ({ + name: droplet.name, + host: droplet.networks.v4.find(n => n.type === 'public').ip_address, + username: 'root', + pem: this.config.sshKey.private, + privateIp: droplet.networks.v4.find(n => n.type === 'private').ip_address, + __droplet: droplet + })).filter(server => { + if (ids) { + return ids.includes(server.__droplet.id); + } + + return true; + }); + } + + async compareServers(servers) { + if (!servers) { + // eslint-disable-next-line no-param-reassign + servers = await this.getServers(); + } + + const good = []; + const wrong = []; + const toResize = []; + + servers.forEach(server => { + const droplet = server.__droplet; + + if (droplet.region.slug !== this.config.region) { + wrong.push(server); + + return; + } + + if ( + droplet.size_slug !== this.config.size + ) { + if (this.config._resize) { + toResize.push(server); + } else { + wrong.push(server); + } + + return; + } + + good.push(server); + }); + + return { + wrong, + good, + toResize + }; + } + + async removeServers(servers) { + const promises = servers.map(server => this._request( + 'delete', + `droplets/${server.__droplet.id}` + )); + + await Promise.all(promises); + } + + async createServers(count) { + let fingerprint = await this._setupPublicKey(); + const names = []; + + let size = this.config.size; + if (this.config._resize) { + size = this.config._resize.initialSize; + } + + while (names.length < count) { + names.push(generateName(this.name)); + } + + const data = { + names, + region: this.config.region, + size, + + // TODO: pick image from API + image: 'ubuntu-20-04-x64', + + // eslint-disable-next-line camelcase + ssh_keys: [ + fingerprint + ], + monitoring: true, + tags: [ + this.tag + ] + }; + + const result = await this._request( + 'post', + 'droplets', + data, + ); + + const ids = result.data.droplets.map(droplet => droplet.id); + await Promise.all(ids.map(id => this._waitForStatus(id, 'active'))); + + if (size !== this.config.size) { + await Promise.all(ids.map(id => this.resizeServer(id, this.config.size))); + } + + return this.getServers(ids); + } + + async _setupPublicKey() { + let content = fs.readFileSync(this.publicKeyPath, 'utf-8'); + let fingerprint = createFingerprint(content); + + try { + await this._request( + 'get', + `account/keys/${fingerprint}` + ); + + return fingerprint; + } catch (e) { + if (!e || !e.response || !e.response.status === 404) { + console.dir(e); + + throw e; + } + + // Key doesn't exist. Ignore error since we will be adding it + } + + try { + await this._request( + 'post', + 'account/keys', + { + // eslint-disable-next-line camelcase + public_key: content, + name: this.config.sshKey.name || this.name + } + ); + } catch (e) { + console.dir(e); + throw e; + } + + return fingerprint; + } + + // Default timeout is 10 minutes + async _waitForStatus(dropletId, desiredStatus, timeout = 1000 * 60 * 10) { + const timeoutAt = Date.now() + timeout; + + while (Date.now() < timeoutAt) { + const response = await this._request( + 'get', + `droplets/${dropletId}` + ); + const status = response.data.droplet.status; + + if (status === desiredStatus) { + return; + } + + await new Promise(resolve => setTimeout(resolve, 1000 * 10)); + } + + throw new Error(`Timed out waiting for droplet ${dropletId} to become active`); + } + + async resizeServer(dropletId) { + await this._shutdownDroplet(dropletId); + + const result = await this._request( + 'post', + `droplets/${dropletId}/actions`, + { + type: 'resize', + disk: false, + size: this.config.size + } + ); + + const actionId = result.data.action.id; + + let timeoutAt = Date.now() + (1000 * 60 * 10); + while (Date.now() < timeoutAt) { + const { data } = await this._request( + 'get', + `droplets/${dropletId}/actions/${actionId}` + ); + + if (data.action.status === 'completed') { + break; + } + + await new Promise(resolve => setTimeout(resolve, 1000 * 10)); + } + + await this._request( + 'post', + `droplets/${dropletId}/actions`, + { + type: 'power_on' + } + ); + + await this._waitForStatus(dropletId, 'active'); + } + + async _shutdownDroplet(dropletId) { + await this._request( + 'post', + `droplets/${dropletId}/actions`, + { + type: 'shutdown' + } + ); + + try { + await this._waitForStatus(dropletId, 'off', 1000 * 60 * 3); + } catch (e) { + console.log(e); + await this._request( + 'post', + `droplets/${dropletId}/actions`, + { + type: 'power_off' + } + ); + await this._waitForStatus(dropletId, 'off'); + } + } + + _request(method, path, data) { + return axios({ + method, + url: `https://api.digitalocean.com/v2/${path}`, + data, + headers: { + Authorization: `Bearer ${this.config.token}` + } + }).catch(err => { + if (err.config && err.config.headers) { + err.config.headers.Authorization = ''; + } + + throw err; + }); + } +} diff --git a/src/plugins/default/server-groups/index.js b/src/plugins/default/server-groups/index.js new file mode 100644 index 00000000..9a4d2410 --- /dev/null +++ b/src/plugins/default/server-groups/index.js @@ -0,0 +1,58 @@ +import DigitalOcean from './digital-ocean'; +import { waitForServers } from './utils'; + +function createSourceConfig(SourceAPI) { + return { + async load({ name, groupConfig }, pluginApi) { + const api = new SourceAPI(name, groupConfig, pluginApi); + + return api.getServers(); + }, + async upToDate({ name, groupConfig, list }, pluginApi) { + const api = new SourceAPI(name, groupConfig, pluginApi); + const { wrong, good, toResize = [] } = await api.compareServers(list); + + return wrong.length === 0 && + good.length === groupConfig.count && + toResize.length === 0; + }, + async update({ name, groupConfig }, pluginApi) { + const api = new SourceAPI(name, groupConfig, pluginApi); + const { wrong, good, toResize = [] } = await api.compareServers(); + + const addCount = Math.max(0, groupConfig.count - good.length); + const goodRemoveCount = Math.max(0, good.length - groupConfig.count); + + if (goodRemoveCount > 0) { + console.log(`=> Removing ${goodRemoveCount} servers for ${name}`); + await api.removeServers(good.slice(0, goodRemoveCount)); + console.log(`=> Finished removing ${goodRemoveCount} servers for ${name}`); + } + if (wrong.length > 0) { + // TODO: don't delete tempCount until mup successfully exits + console.log(`=> Removing ${wrong.length} servers for ${name}`); + await api.removeServers(wrong); + console.log(`=> Finished removing ${wrong.length} servers for ${name}`); + } + if (addCount > 0) { + console.log(`=> Creating ${addCount} servers for ${name}`); + const created = await api.createServers(addCount); + await waitForServers(created, pluginApi); + console.log(`=> Finished creating ${addCount} servers for ${name}`); + } + + for (const server of toResize) { + console.log(`=> Resizing ${server.__droplet.name} for ${name}`); + await api.resizeServer(server.__droplet.id); + await waitForServers([server], pluginApi); + console.log(`=> Finished resizing ${server.__droplet.name} for ${name}`); + } + } + }; +} + +const serverSources = { + 'digital-ocean': createSourceConfig(DigitalOcean) +}; + +export default serverSources; diff --git a/src/plugins/default/server-groups/utils.js b/src/plugins/default/server-groups/utils.js new file mode 100644 index 00000000..329b836e --- /dev/null +++ b/src/plugins/default/server-groups/utils.js @@ -0,0 +1,56 @@ +import crypto from 'crypto'; +import { Client } from 'ssh2-classic'; + +export function generateName(groupName) { + const randomString = crypto.randomBytes(4).toString('hex'); + + return `${groupName}-${randomString}`; +} + +export function createFingerprint(keyContent) { + const cleanedContent = keyContent + // Remove key type at beginning + .replace(/(^ssh-[a-zA-Z0-9]*)/, '') + .trim() + // Remove comment at end + .replace(/ [^ ]+$/, ''); + + const buffer = Buffer.from(cleanedContent, 'base64'); + const hash = crypto.createHash('md5').update(buffer).digest('hex'); + + // Add colons between every 2 characters + return hash.match(/.{1,2}/g).join(':'); +} + +const FIVE_MINUTES = 1000 * 60 * 5; +export function waitForServers(servers, api) { + async function waitForServer(server, startTime = Date.now()) { + if (Date.now() - startTime > FIVE_MINUTES) { + throw new Error('Timed out waiting for server to accept SSH connections'); + } + + try { + await new Promise((resolve, reject) => { + const ssh = api._createSSHOptions(server); + const conn = new Client(); + conn.once('error', err => { + reject(err); + }).once('ready', () => { + conn.end(); + resolve(); + }).connect(ssh); + }); + } catch (e) { + if (e.code !== 'ECONNREFUSED') { + console.dir(e); + } + await new Promise(resolve => setTimeout(resolve, 1000 * 5)); + + return waitForServer(server); + } + } + + return Promise.all( + servers.map(server => waitForServer(server)) + ); +} diff --git a/src/plugins/default/template/mup.js.sample b/src/plugins/default/template/mup.js.sample index 5c9d9a09..da66503f 100644 --- a/src/plugins/default/template/mup.js.sample +++ b/src/plugins/default/template/mup.js.sample @@ -1,17 +1,23 @@ module.exports = { servers: { one: { - // TODO: set host address, username, and authentication method + // TODO: set host address and username host: '1.2.3.4', username: 'root', - // pem: './path/to/pem' + + // TODO: Choose one of these authentication methods + // - To use a private SSH key, set the `pem` option to the path to the key + // - For a password, set the `password` option to the user's password + // - If neither `pem` or `password` are set, the ssh-agent is used + // for authentication. + // + // pem: '~/.ssh/key-name' // password: 'server-password' - // or neither for authenticate from ssh-agent } }, app: { - // TODO: change app name and path + // TODO: change app name and the path to the app name: 'app', path: '', @@ -20,24 +26,20 @@ module.exports = { }, buildOptions: { + // Set to true to skip building mobile apps serverOnly: true, }, env: { - // TODO: Change to your app's url - // If you are using ssl, it needs to start with https:// + // TODO: Change to your app's url. This should be + // the url you access your app at, including the correct + // protocol (https:// or http://) and, if you've changed it + // from the default, the port. ROOT_URL: 'http://app.com', + MONGO_URL: 'mongodb://mongodb/meteor', MONGO_OPLOG_URL: 'mongodb://mongodb/local', }, - - docker: { - image: 'zodern/meteor:root', - }, - - // Show progress bar while uploading bundle to server - // You might need to disable it on CI servers - enableUploadProgressBar: true }, mongo: { @@ -48,15 +50,22 @@ module.exports = { }, // (Optional) - // Use the proxy to setup ssl or to route requests to the correct - // app when there are several apps - + // Use the proxy to: + // - setup ssl + // - if multiple apps share the server, route requests to the correct one + // - load balance requests across multiple servers + // // proxy: { // domains: 'mywebsite.com,www.mywebsite.com', // ssl: { // // Enable Let's Encrypt - // letsEncryptEmail: 'email@domain.com' - // } + // // This email is used by Let's encrypt to + // // inform you when a certificate is close to + // // expiring. + // letsEncryptEmail: 'email@domain.com', + // }, + // + // loadBalancing: true // } }; diff --git a/src/plugins/docker/assets/docker-setup.sh b/src/plugins/docker/assets/docker-setup.sh index 2833e5a8..2f82585d 100644 --- a/src/plugins/docker/assets/docker-setup.sh +++ b/src/plugins/docker/assets/docker-setup.sh @@ -1,4 +1,5 @@ #!/bin/bash +exec 2>&1 # TODO make sure we can run docker in this server @@ -18,8 +19,17 @@ echo "Major" $majorVersion echo "Minor" $minorVersion set -e -if [ ! "$hasDocker" ]; then +retry_install () { + echo "waiting 30 seconds" + sleep 30 + echo "trying installation again" install_docker +} + +if [ ! "$hasDocker" ]; then + # If the server was just created, it might still be running some apt-get + # commands and installing docker could fail due to apt-get locks. + install_docker || retry_install || retry_install elif [ "$minimumMajor" -gt "$majorVersion" ]; then echo "major wrong" diff --git a/src/plugins/docker/command-handlers.js b/src/plugins/docker/command-handlers.js index 327b2073..825047e1 100644 --- a/src/plugins/docker/command-handlers.js +++ b/src/plugins/docker/command-handlers.js @@ -21,7 +21,7 @@ import nodemiral from '@zodern/nodemiral'; const log = debug('mup:module:docker'); -function uniqueSessions(api) { +export function uniqueSessions(api) { const { servers } = api.getConfig(); const sessions = api.getSessions(['app', 'mongo', 'proxy']); @@ -248,8 +248,11 @@ export async function status(api) { return; } + const allServers = Object.values(api.expandServers(config.servers || {})) + .map(({ server }) => server); + const results = await map( - Object.values(config.servers), + allServers, server => api.runSSHCommand(server, 'sudo docker version --format "{{.Server.Version}}"'), { concurrency: 2 } ); diff --git a/src/plugins/docker/index.js b/src/plugins/docker/index.js index 9b8ac4a8..84318327 100644 --- a/src/plugins/docker/index.js +++ b/src/plugins/docker/index.js @@ -1,3 +1,4 @@ +import { uniqueSessions } from './command-handlers'; import * as _commands from './commands'; import { validateRegistry, validateSwarm } from './validate'; @@ -50,3 +51,28 @@ export function scrubConfig(config) { return config; } + +export async function checkSetup(api) { + const sessions = await uniqueSessions(api); + const config = api.getConfig(); + + return [ + { + sessions, + name: 'docker', + setupKey: { + scripts: [ + api.resolvePath(__dirname, 'assets/docker-setup.sh'), + api.resolvePath(__dirname, 'assets/install-docker.sh') + ], + config: { + privateDockerRegistry: config.privateDockerRegistry, + // TODO: fix this to avoid always needing to setup everytime when + // swarm is enabled + swarm: api.swarmEnabled() ? Date.now() : false + } + }, + services: ['docker'] + } + ]; +} diff --git a/src/plugins/meteor/assets/clean-versions.sh b/src/plugins/meteor/assets/clean-versions.sh new file mode 100644 index 00000000..ad655bc3 --- /dev/null +++ b/src/plugins/meteor/assets/clean-versions.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +APP_NAME=<%= appName %> +IMAGE=<%= imagePrefix %><%= appName.toLowerCase() %> + +<% for(version of versions) { %> + docker rmi $IMAGE:<%- version %> +<% } %> diff --git a/src/plugins/meteor/assets/meteor-deploy-check.sh b/src/plugins/meteor/assets/meteor-deploy-check.sh index c1808853..749356d2 100644 --- a/src/plugins/meteor/assets/meteor-deploy-check.sh +++ b/src/plugins/meteor/assets/meteor-deploy-check.sh @@ -1,4 +1,5 @@ #!/bin/bash +exec 2>&1 APPNAME=<%= appName %> APP_PATH=/opt/$APPNAME @@ -15,39 +16,64 @@ cd $APP_PATH revert_app () { echo "=> Container status:" sudo docker inspect $APPNAME --format "restarted: {{.RestartCount}} times {{json .NetworkSettings}} {{json .State}}" - echo "=> Logs:" 1>&2 + echo "" + echo "" + echo "==================================" + echo "=> App Logs:" 1>&2 sudo docker logs --tail=100 $APPNAME 1>&2 - <% if (privateRegistry) { %> - sudo docker pull $IMAGE:previous || true + <% if (canRollback) { %> + # Record that the version failed + FAILED_VERSION=$(tail -1 /opt/$APPNAME/config/version-history.txt) || '' + PREVIOUS_VERSION=$(tail -n 2 /opt/$APPNAME/config/version-history.txt | head -1) || '' + + # Make sure we can roll back using a versioned image + if [ -n "$FAILED_VERSION" ] && [ -n "$PREVIOUS_VERSION" ]; then + <% if (recordFailed) { %> + # TODO: limit how large this file can grow + echo "$FAILED_VERSION" >> /opt/$APPNAME/config/failed-versions.txt + <% } %> + + # Remove last line so we don't rollback to the failed version + head -n -1 /opt/$APPNAME/config/version-history.txt > /opt/$APPNAME/config/version-history.tmp.txt + mv /opt/$APPNAME/config/version-history.tmp.txt /opt/$APPNAME/config/version-history.txt + + sudo bash $START_SCRIPT > /dev/null 2>&1 + echo " " 1>&2 + echo "=> Redeploying previous version of the app" 1>&2 + echo " " 1>&2 + + # If the versioned image isn't available, check if there is a previous image + elif sudo docker image inspect $IMAGE:latest >/dev/null; then + sudo docker tag $IMAGE:previous $IMAGE:latest + + <% if (privateRegistry) { %> + sudo docker push $IMAGE:latest + docker image prune -f + <% } %> + + echo "latest" > /opt/$APPNAME/config/version-history.txt + sudo bash $START_SCRIPT > /dev/null 2>&1 + + echo " " 1>&2 + echo "=> Redeploying previous version of the app" 1>&2 + echo " " 1>&2 + + elif [ -d last ]; then + rm /opt/$APPNAME/config/version-history.txt + sudo mv last current + sudo bash $START_SCRIPT > /dev/null 2>&1 + + echo " " 1>&2 + echo "=> Redeploying previous version of the app" 1>&2 + echo " " 1>&2 + fi <% } %> - - if sudo docker image inspect $IMAGE:previous >/dev/null 2>&1; then - sudo docker tag $IMAGE:previous $IMAGE:latest - - <% if (privateRegistry) { %> - sudo docker push $IMAGE:latest - docker image prune -f - <% } %> - - sudo bash $START_SCRIPT > /dev/null 2>&1 - - echo " " 1>&2 - echo "=> Redeploying previous version of the app" 1>&2 - echo " " 1>&2 - - elif [ -d last ]; then - sudo mv last current - sudo bash $START_SCRIPT > /dev/null 2>&1 - - echo " " 1>&2 - echo "=> Redeploying previous version of the app" 1>&2 - echo " " 1>&2 - fi +} - echo - echo "To see more logs type 'mup logs --tail=200'" - echo "" +finish () { + # TODO: if canRollback is true, remove version from failed list + exit 0 } START_TIME=$(date +%s) diff --git a/src/plugins/meteor/assets/meteor-start.sh b/src/plugins/meteor/assets/meteor-start.sh index feb03243..46bb9798 100644 --- a/src/plugins/meteor/assets/meteor-start.sh +++ b/src/plugins/meteor/assets/meteor-start.sh @@ -6,6 +6,11 @@ APP_DIR=/opt/<%=appName %> IMAGE=mup-<%= appName.toLowerCase() %> PRIVATE_REGISTRY=<%- privateRegistry ? 0 : 1 %> +<% if (typeof version === 'number') { %> + # TODO: limit how large this file can grow + echo "<%= version %>" >> $APP_DIR/config/version-history.txt +<% } %> + <% if (removeImage) { %> echo "Removing images" # Run when the docker image doesn't support prepare-bundle.sh. @@ -13,13 +18,15 @@ echo "Removing images" # Removes the latest image, so the start script will use the bundle instead sudo docker rmi $IMAGE:latest || true sudo docker images +rm $APP_DIR/config/version-history.txt || true <% } %> # save the last known version cd $APP_DIR -if sudo docker image inspect $IMAGE:latest >/dev/null || [ "$PRIVATE_REGISTRY" == "0" ]; then +if sudo docker image inspect $IMAGE:latest >/dev/null || [ "$PRIVATE_REGISTRY" == "0" ] || [ -f $APP_DIR/config/version-history.txt ]; then echo "using image" sudo rm -rf current || true + else echo "using bundle" sudo rm -rf last @@ -32,8 +39,7 @@ else sudo docker rmi $IMAGE:previous || true fi -# TODO: clean up the last folder when the private registry has a previous image -if sudo docker image inspect $IMAGE:previous >/dev/null; then +if sudo docker image inspect $IMAGE:previous >/dev/null || [[ $(wc -l < $APP_DIR/config/version-history.txt) -ge 2 ]] ; then echo "removing last" sudo rm -rf last fi diff --git a/src/plugins/meteor/assets/meteor-stop.sh b/src/plugins/meteor/assets/meteor-stop.sh index 9c7bea4c..c2e48448 100644 --- a/src/plugins/meteor/assets/meteor-stop.sh +++ b/src/plugins/meteor/assets/meteor-stop.sh @@ -2,6 +2,7 @@ APPNAME=<%= appName %> +sudo docker stop -t 30 $APPNAME || : sudo docker rm -f $APPNAME || : sudo docker rm -f $APPNAME-frontend || : sudo docker rm -f $APPNAME-nginx-letsencrypt || : diff --git a/src/plugins/meteor/assets/prepare-bundle.sh b/src/plugins/meteor/assets/prepare-bundle.sh index 05e3672c..a6079401 100644 --- a/src/plugins/meteor/assets/prepare-bundle.sh +++ b/src/plugins/meteor/assets/prepare-bundle.sh @@ -1,4 +1,5 @@ #!/bin/bash +exec 2>&1 exec 2>&1 set -e @@ -86,7 +87,7 @@ sudo docker tag $IMAGE:build $IMAGE:<%= tag %> # Fails if the previous tag doesn't exist (such as during the initial deploy) sudo docker push $IMAGE:previous || true - sudo docker push $IMAGE:latest + sudo docker push $IMAGE:<%= tag %> <% } %> diff --git a/src/plugins/meteor/assets/templates/start.sh b/src/plugins/meteor/assets/templates/start.sh index bea847a6..6d9fd055 100644 --- a/src/plugins/meteor/assets/templates/start.sh +++ b/src/plugins/meteor/assets/templates/start.sh @@ -10,14 +10,19 @@ BIND=<%= bind %> NGINX_PROXY_VERSION="v1.1.0" LETS_ENCRYPT_VERSION="v1.13.1" APP_IMAGE=<%- imagePrefix %><%= appName.toLowerCase() %> -IMAGE=$APP_IMAGE:latest + +TAG=$(tail -1 $APP_PATH/config/version-history.txt) +TAG=${TAG:="latest"} +echo "TAG: $TAG" + +IMAGE=$APP_IMAGE:$TAG VOLUME="--volume=$BUNDLE_PATH:/bundle" LOCAL_IMAGE=false <% if (!privateRegistry) { %> sudo docker image inspect $IMAGE >/dev/null || IMAGE=<%= docker.image %> -if [ $IMAGE == $APP_IMAGE:latest ]; then +if [ $IMAGE == $APP_IMAGE:$TAG ]; then VOLUME="" LOCAL_IMAGE=true fi diff --git a/src/plugins/meteor/command-handlers.js b/src/plugins/meteor/command-handlers.js index 4658d73d..c417880b 100644 --- a/src/plugins/meteor/command-handlers.js +++ b/src/plugins/meteor/command-handlers.js @@ -3,11 +3,11 @@ import { checkAppStarted, createEnv, createServiceConfig, - currentImageTag, escapeEnvQuotes, getImagePrefix, getNodeVersion, getSessions, + getVersions, shouldRebuild } from './utils'; import buildApp, { archiveApp, cleanBuildDir } from './build.js'; @@ -16,6 +16,9 @@ import { map, promisify } from 'bluebird'; import { prepareBundleLocally, prepareBundleSupported } from './prepare-bundle'; import debug from 'debug'; import nodemiral from '@zodern/nodemiral'; +import { rollback } from './rollback'; +import state from './state'; +import { Client } from 'ssh2-classic'; const log = debug('mup:module:meteor'); @@ -142,51 +145,79 @@ export async function prepareBundle(api) { const buildOptions = appConfig.buildOptions; const bundlePath = api.resolvePath(buildOptions.buildLocation, 'bundle.tar.gz'); + // await getVersions(api); + + const sessions = api.getSessions(['app']); + + const { latest, servers: serverVersions } = await getVersions(api); + const tag = latest + 1; + state.deployingVersion = tag; + if (appConfig.docker.prepareBundleLocally) { - return prepareBundleLocally(buildOptions.buildLocation, bundlePath, api); - } + await prepareBundleLocally(buildOptions.buildLocation, bundlePath, api); + } else { + const list = nodemiral.taskList('Prepare App Bundle'); + const nodeVersion = await getNodeVersion(bundlePath); + + list.executeScript('Prepare Bundle', { + script: api.resolvePath( + __dirname, + 'assets/prepare-bundle.sh' + ), + vars: { + appName: appConfig.name, + dockerImage: appConfig.docker.image, + env: escapeEnvQuotes(appConfig.env), + buildInstructions: appConfig.docker.buildInstructions || [], + nodeVersion, + stopApp: appConfig.docker.stopAppDuringPrepareBundle, + useBuildKit: appConfig.docker.useBuildKit, + tag, + privateRegistry: privateDockerRegistry, + imagePrefix: getImagePrefix(privateDockerRegistry) + } + }); - const list = nodemiral.taskList('Prepare App Bundle'); + // After running Prepare Bundle, the list of images will be out of date + api.serverInfoStale(); - let tag = 'latest'; + let prepareSessions = sessions; + if (privateDockerRegistry) { + prepareSessions = [sessions[0]].filter(s => s); + } - if (api.swarmEnabled()) { - const data = await api.getServerInfo(); - tag = currentImageTag(data, appConfig.name) + 1; + await api.runTaskList(list, prepareSessions, { + series: true, + verbose: api.verbose + }); } - const nodeVersion = await getNodeVersion(bundlePath); + const toClean = Object.create(null); - list.executeScript('Prepare Bundle', { - script: api.resolvePath( - __dirname, - 'assets/prepare-bundle.sh' - ), + serverVersions.forEach(({ host, versions, current, previous }) => { + let toKeep = [current, previous, tag]; + toClean[host] = { + versions: versions.filter(version => !toKeep.includes(version)) + }; + }); + + + const list = nodemiral.taskList('Clean Up Versions'); + + list.executeScript('Clean up app versions', { + script: api.resolvePath(__dirname, 'assets/clean-versions.sh'), vars: { + // TODO: add a default version-history from other servers + // on servers that don't have a history so it has something to + // rollback to if the current deploy fails appName: appConfig.name, - dockerImage: appConfig.docker.image, - env: escapeEnvQuotes(appConfig.env), - buildInstructions: appConfig.docker.buildInstructions || [], - nodeVersion, - stopApp: appConfig.docker.stopAppDuringPrepareBundle, - useBuildKit: appConfig.docker.useBuildKit, - tag, - privateRegistry: privateDockerRegistry, imagePrefix: getImagePrefix(privateDockerRegistry) - } + }, + hostVars: toClean }); - // After running Prepare Bundle, the list of images will be out of date - api.serverInfoStale(); - - let sessions = api.getSessions(['app']); - if (privateDockerRegistry) { - sessions = sessions.slice(0, 1); - } - - return api.runTaskList(list, sessions, { - series: true, - verbose: api.verbose + await api.runTaskList(list, sessions, { + series: false }); } @@ -221,7 +252,7 @@ export async function push(api) { list.copy('Pushing Meteor App Bundle to the Server', { src: bundlePath, dest: `/opt/${appConfig.name}/tmp/bundle.tar.gz`, - progressBar: appConfig.enableUploadProgressBar + progressBar: true }); let sessions = api.getSessions(['app']); @@ -244,7 +275,6 @@ export async function push(api) { export function envconfig(api) { log('exec => mup meteor envconfig'); const { - servers, app, proxy, privateDockerRegistry @@ -288,15 +318,16 @@ export function envconfig(api) { } const startHostVars = {}; + const expandedServers = api.expandServers(app.servers); - Object.keys(app.servers).forEach(serverName => { - const host = servers[serverName].host; + Object.values(expandedServers).forEach(({ server, config }) => { + const host = server.host; const vars = {}; - if (app.servers[serverName].bind) { - vars.bind = app.servers[serverName].bind; + if (config.bind) { + vars.bind = config.bind; } - if (app.servers[serverName].env && app.servers[serverName].env.PORT) { - vars.port = app.servers[serverName].env.PORT; + if (config.env && config.env.PORT) { + vars.port = config.env.PORT; } startHostVars[host] = vars; }); @@ -325,20 +356,20 @@ export function envconfig(api) { const env = createEnv(app, api.getSettings()); const hostVars = {}; - Object.keys(app.servers).forEach(key => { - const host = servers[key].host; - if (app.servers[key].env) { + Object.values(expandedServers).forEach(({ server, config }) => { + const host = server.host; + if (config.env) { hostVars[host] = { env: { - ...app.servers[key].env, + ...config.env, // We treat the PORT specially and do not pass it to the container PORT: undefined } }; } - if (app.servers[key].settings) { + if (config.settings) { const settings = JSON.stringify(api.getSettingsFromPath( - app.servers[key].settings)); + config.settings)); if (hostVars[host]) { hostVars[host].env.METEOR_SETTINGS = settings; @@ -368,7 +399,7 @@ export function envconfig(api) { export async function start(api) { log('exec => mup meteor start'); - const config = api.getConfig().app; + const { app: config } = api.getConfig(); const swarmEnabled = api.swarmEnabled(); if (!config) { @@ -376,12 +407,14 @@ export async function start(api) { process.exit(1); } + const isDeploy = api.commandHistory.find(entry => + ['meteor.deploy', 'meteor.deployVersion'].includes(entry.name) + ); const list = nodemiral.taskList('Start Meteor'); if (swarmEnabled) { const currentService = await api.dockerServiceInfo(config.name); - const serverInfo = await api.getServerInfo(); - const imageTag = currentImageTag(serverInfo, config.name); + const { latest: imageTag } = await getVersions(api); // TODO: make it work when the reverse proxy isn't enabled api.tasks.addCreateOrUpdateService( @@ -390,16 +423,39 @@ export async function start(api) { currentService ); } else { - addStartAppTask(list, api); - checkAppStarted(list, api); + addStartAppTask(list, api, { isDeploy, version: state.deployingVersion }); + checkAppStarted(list, api, { + canRollback: isDeploy, + recordFailed: isDeploy && !api.commandHistory.find( + entry => entry.name === 'meteor.deployVersion' + ) + }); } const sessions = await getSessions(api); - return api.runTaskList(list, sessions, { - series: true, - verbose: api.verbose - }); + try { + await api.runTaskList(list, sessions, { + series: true, + verbose: api.verbose + }); + + if (isDeploy) { + console.log(`Successfully deployed version ${state.deployingVersion}`); + } + } catch (e) { + if ( + isDeploy && + prepareBundleSupported(config.docker) && + !api.swarmEnabled() + ) { + console.log('Deploy failed. Check the logs above for the reason'); + console.log('=> Ensuring all servers have same version'); + await rollback(api); + } + + throw e; + } } export function deploy(api) { @@ -419,6 +475,37 @@ export function deploy(api) { .then(() => api.runCommand('default.reconfig')); } +export async function deployVersion(api) { + log('exec => mup meteor deploy'); + + // validate settings and config before starting + api.getSettings(); + const config = api.getConfig().app; + + if (!config) { + console.error('error: no configs found for meteor'); + process.exit(1); + } + + let version = api.getArgs()[2]; + + if (!version) { + console.error('Please provide a version'); + process.exit(1); + } + + version = parseInt(version, 10); + + if (Number.isNaN(version)) { + console.log('Version is not a valid number'); + process.exit(1); + } + + state.deployingVersion = version; + + return api.runCommand('default.reconfig'); +} + export async function stop(api) { log('exec => mup meteor stop'); const config = api.getConfig().app; @@ -459,6 +546,9 @@ export async function restart(api) { if (api.swarmEnabled()) { api.tasks.addRestartService(list, { name: appConfig.name }); } else { + list._runHook('Stopping app', { + hookName: 'app.shutdown' + }); list.executeScript('Stop Meteor', { script: api.resolvePath(__dirname, 'assets/meteor-stop.sh'), vars: { @@ -467,6 +557,9 @@ export async function restart(api) { }); addStartAppTask(list, api); checkAppStarted(list, api); + list._runHook('Finish starting', { + hookName: 'app.start-instance' + }); } @@ -478,21 +571,21 @@ export async function restart(api) { export async function debugApp(api) { const { - servers, app } = api.getConfig(); let serverOption = api.getArgs()[2]; + let expandedServers = api.expandServers(app.servers); // Check how many sessions are enabled. Usually is all servers, // but can be reduced by the `--servers` option const enabledSessions = api.getSessions(['app']) .filter(session => session); - if (!(serverOption in app.servers)) { + if (!(serverOption in expandedServers)) { if (enabledSessions.length === 1) { const selectedHost = enabledSessions[0]._host; - serverOption = Object.keys(app.servers).find( - name => servers[name].host === selectedHost + serverOption = Object.keys(expandedServers).find( + name => expandedServers[name].server.host === selectedHost ); } else { console.log('mup meteor debug '); @@ -503,7 +596,7 @@ export async function debugApp(api) { } } - const server = servers[serverOption]; + const server = expandedServers[serverOption].server; console.log(`Setting up to debug app running on ${serverOption}`); const { @@ -578,6 +671,61 @@ export async function debugApp(api) { }); } +export async function meteorShell(api) { + const { app } = api.getConfig(); + const expandedServers = api.expandServers(app.servers); + let serverOption = api.getArgs()[1]; + + // Check how many sessions are enabled. Usually is all servers, + // but can be reduced by the `--servers` option + const enabledSessions = api.getSessionsForServers(Object.keys(app.servers)) + .filter(session => session); + + if (!(serverOption in expandedServers)) { + if (enabledSessions.length === 1) { + const selectedHost = enabledSessions[0]._host; + serverOption = Object.keys(expandedServers).find(key => + expandedServers[key].server.host === selectedHost); + } else { + console.log('mup meteor shell '); + console.log('Available servers are:\n', Object.keys(expandedServers).join('\n ')); + process.exitCode = 1; + + return; + } + } + + const server = expandedServers[serverOption].server; + const sshOptions = api._createSSHOptions(server); + + const conn = new Client(); + conn.on('ready', () => { + conn.exec( + `docker exec -it ${app.name} node ./meteor-shell.js`, + { pty: true }, + (err, stream) => { + if (err) { + throw err; + } + stream.on('close', () => { + conn.end(); + process.exit(); + }); + + process.stdin.setRawMode(true); + process.stdin.pipe(stream); + + stream.pipe(process.stdout); + stream.stderr.pipe(process.stderr); + stream.setWindow(process.stdout.rows, process.stdout.columns); + + process.stdout.on('resize', () => { + stream.setWindow(process.stdout.rows, process.stdout.columns); + }); + }); + }).connect(sshOptions); +} + export async function destroy(api) { const config = api.getConfig(); const options = api.getOptions(); @@ -626,10 +774,12 @@ export async function status(api) { StatusDisplay } = api.statusHelpers; const overview = api.getOptions().overview; - const servers = Object.keys(config.app.servers) + const expandedServers = api.expandServers(config.app.servers); + const servers = Object.keys(expandedServers) .map(key => ({ - ...config.servers[key], - name: key + ...expandedServers[key].server, + name: key, + overrides: expandedServers[key].config })); const results = await map( @@ -665,3 +815,32 @@ export async function status(api) { display.show(overview); } + +export async function listVersions(api) { + const versions = await getVersions(api); + console.log('Application versions:'); + // TODO: when using private docker registry, combine versions + // and history to get a more complete list + versions.versions.forEach(version => { + let text = ` - ${version}`; + + if (version === versions.current) { + text += ' (current)'; + } else if (version === versions.previous) { + text += ' (previous)'; + } else if (versions.failed.includes(version)) { + text += ' (failed)'; + } + + text = text.padEnd(17, ' '); + + let date = versions.versionDates.get(version); + text += ` created ${date.toLocaleDateString('en-US', { dateStyle: 'short', timeStyle: 'short' })}`; + + console.log(text); + }); + + console.log(); + console.log('Switch to a different version by running:'); + console.log(' mup meteor deploy-version '); +} diff --git a/src/plugins/meteor/commands.js b/src/plugins/meteor/commands.js index 9dcacaf0..966d47d4 100644 --- a/src/plugins/meteor/commands.js +++ b/src/plugins/meteor/commands.js @@ -87,6 +87,29 @@ export const debug = { handler: commandHandlers.debugApp }; +export const shell = { + name: 'shell [server]', + description: 'Open production Meteor shell', + builder(yargs) { + yargs.strict(false); + }, + handler: commandHandlers.meteorShell +}; + +export const versions = { + description: 'List application versions', + handler: commandHandlers.listVersions +}; + +export const deployVersion = { + name: 'deploy-version [version]', + description: 'Deploy specific application version', + builder(yargs) { + yargs.strict(false); + }, + handler: commandHandlers.deployVersion +}; + // Hidden commands export const build = { description: false, diff --git a/src/plugins/meteor/index.js b/src/plugins/meteor/index.js index 14cdc1a8..977a5b40 100644 --- a/src/plugins/meteor/index.js +++ b/src/plugins/meteor/index.js @@ -1,7 +1,7 @@ import * as _commands from './commands'; import _validator from './validate'; import { defaultsDeep } from 'lodash'; -import { tmpBuildPath } from './utils'; +import { getSessions, tmpBuildPath } from './utils'; import traverse from 'traverse'; export const description = 'Deploy and manage meteor apps'; @@ -27,8 +27,9 @@ export function prepareConfig(config, api) { } config.app.docker = defaultsDeep(config.app.docker, { - image: config.app.dockerImage || 'kadirahq/meteord', - stopAppDuringPrepareBundle: true + image: config.app.dockerImage || 'zodern/meteor:0.6.1-root', + stopAppDuringPrepareBundle: true, + useBuildKit: true }); delete config.app.dockerImage; @@ -138,3 +139,28 @@ export function swarmOptions(config) { }; } } + +export async function checkSetup(api) { + const config = api.getConfig(); + if (!config.app || config.app.type !== 'meteor') { + return []; + } + + const sessions = await getSessions(api); + + return [ + { + sessions, + name: `meteor-${config.app.name}`, + setupKey: { + // TODO: handle legacy ssl configuration + scripts: [ + api.resolvePath(__dirname, 'assets/meteor-setup.sh') + ], + config: { + name: config.app.name + } + } + } + ]; +} diff --git a/src/plugins/meteor/rollback.js b/src/plugins/meteor/rollback.js new file mode 100644 index 00000000..98c877f6 --- /dev/null +++ b/src/plugins/meteor/rollback.js @@ -0,0 +1,103 @@ +import state from './state'; +import { addStartAppTask, checkAppStarted, getSessions, getVersions } from './utils'; + +// After a failed deploy, ensure all servers are running the same version +export async function rollback(api) { + const { + app: appConfig, + privateDockerRegistry + } = api.getConfig(); + const sessions = await getSessions(api); + const versions = await getVersions(api); + + if (versions.servers.length === 1) { + // There are no other servers to make sure it is consistent with + return; + } + + // The servers that are running the new version, and are able to rollback + // to the previous version + const toRollback = versions.servers.filter(server => { + if (server.current !== state.deployingVersion) { + return false; + } + + if (server.previous && privateDockerRegistry) { + return true; + } + + // Make sure the server has the previous version + return server.versions.includes(server.previous); + }); + + function getSession(server) { + return sessions.find(session => session._host === server.host); + } + + // TODO: when there two servers, make sure the app is available on the other + // server first. Otherwise, we might restart the only running instance of the + // app, causing downtime + + // TODO: if all servers have the same current version, do nothing since + // it indicates the meteor-deploy-check script never never rolled back to + // the old version (we don't know if the app version is bad, or there was + // a connection issue or some other problem with the last server). + + let list = new QuietList(); + for (const server of toRollback) { + const session = getSession(server); + if (session) { + // Remove the failed version from history so we don't roll back to it + await api.runSSHCommand( + session, + [ + `head -n -1 /opt/${appConfig.name}/config/version-history.txt > /opt/${appConfig.name}/config/version-history.tmp.txt`, + `mv /opt/${appConfig.name}/config/version-history.tmp.txt /opt/${appConfig.name}/config/version-history.txt` + ].join('; ') + ); + let version = server.previous; + addStartAppTask(list, api, { isDeploy: false, version }); + checkAppStarted(list, api); + try { + console.log(` - ${server.name}: rolling back to previous version...`); + await list.run(session); + console.log(` - ${server.name}: rolled back successfully`); + } catch (e) { + console.log(` - ${server.name}: failed rolling back`); + } + } + } +} + +class QuietList { + constructor() { + this.list = []; + } + + executeScript(name, { script, vars }) { + this.list.push({ script, vars }); + } + + _run(session, script, vars) { + return new Promise((resolve, reject) => { + session.executeScript(script, { + vars + }, err => { + if (err) { + reject(err); + } else { + resolve(); + } + }); + }); + } + + async run(session) { + let list = this.list; + this.list = []; + + for (const entry of list) { + await this._run(session, entry.script, entry.vars); + } + } +} diff --git a/src/plugins/meteor/state.js b/src/plugins/meteor/state.js new file mode 100644 index 00000000..226691c8 --- /dev/null +++ b/src/plugins/meteor/state.js @@ -0,0 +1,7 @@ +// State shared between different commands + +const state = { + deployingVersion: null +}; + +export default state; diff --git a/src/plugins/meteor/status.js b/src/plugins/meteor/status.js index 2232f36a..01eea1c5 100644 --- a/src/plugins/meteor/status.js +++ b/src/plugins/meteor/status.js @@ -103,11 +103,10 @@ async function checkUrlLocally(server, appConfig, port) { function getCheckAddress(server, appConfig) { if ( - appConfig.servers && - appConfig.servers[server.name] && - appConfig.servers[server.name].bind + server.overrides && + server.overrides.bind ) { - return appConfig.servers[server.name].bind; + return server.overrides.bind; } if (appConfig.docker && appConfig.docker.bind) { @@ -118,8 +117,8 @@ function getCheckAddress(server, appConfig) { } export async function checkUrls(server, appConfig, api) { - const port = appConfig.servers[server.name].env ? - appConfig.servers[server.name].env.PORT : + const port = server.overrides.env && server.overrides.env.PORT ? + server.overrides.env.PORT : appConfig.env.PORT; const [ diff --git a/src/plugins/meteor/utils.js b/src/plugins/meteor/utils.js index 82a88843..4c5a142a 100644 --- a/src/plugins/meteor/utils.js +++ b/src/plugins/meteor/utils.js @@ -1,5 +1,5 @@ import * as uuid from 'uuid'; -import { cloneDeep, flatMap } from 'lodash'; +import { cloneDeep } from 'lodash'; import fs from 'fs'; import os from 'os'; import { @@ -9,7 +9,7 @@ import random from 'random-seed'; import { spawn } from 'child_process'; import tar from 'tar'; -export function checkAppStarted(list, api) { +export function checkAppStarted(list, api, { canRollback, recordFailed } = {}) { const script = api.resolvePath(__dirname, 'assets/meteor-deploy-check.sh'); const { app, privateDockerRegistry } = api.getConfig(); const publishedPort = app.docker.imagePort || 80; @@ -21,28 +21,28 @@ export function checkAppStarted(list, api) { appName: app.name, deployCheckPort: publishedPort, privateRegistry: privateDockerRegistry, - imagePrefix: getImagePrefix(privateDockerRegistry) + imagePrefix: getImagePrefix(privateDockerRegistry), + canRollback: canRollback || false, + recordFailed: recordFailed || false } }); return list; } -export function addStartAppTask(list, api) { +export function addStartAppTask(list, api, { isDeploy, version } = {}) { const { app: appConfig, privateDockerRegistry } = api.getConfig(); - const isDeploy = api.commandHistory.find( - ({ name }) => name === 'meteor.deploy' - ); list.executeScript('Start Meteor', { script: api.resolvePath(__dirname, 'assets/meteor-start.sh'), vars: { appName: appConfig.name, removeImage: isDeploy && !prepareBundleSupported(appConfig.docker), - privateRegistry: privateDockerRegistry + privateRegistry: privateDockerRegistry, + version: typeof version === 'number' ? version : null } }); @@ -210,17 +210,96 @@ export function getImagePrefix(privateRegistry) { return 'mup-'; } -export function currentImageTag(serverInfo, appName) { - const result = flatMap( - Object.values(serverInfo), - ({images}) => images || [] - ) - .filter(image => image.Repository === `mup-${appName}`) - .map(image => parseInt(image.Tag, 10)) - .filter(tag => !isNaN(tag)) - .sort((a, b) => b - a); - - return result[0] || 0; +function mostCommon(values) { + let counts = new Map(); + values.forEach(value => { + let number = counts.get(value) || 0; + counts.set(value, number + 1); + }); + + if (counts.size === 0) { + return null; + } + + return [...counts].sort((a, b) => b[1] - a[1])[0][0]; +} + +export async function getVersions(api) { + const { + app: appConfig, + privateDockerRegistry + } = api.getConfig(); + + const imageName = `${getImagePrefix(privateDockerRegistry)}${appConfig.name.toLowerCase()}`; + + const collector = { + images: { + command: `sudo docker images ${imageName} --format '{{json .}}'`, + parser: 'jsonArray' + }, + history: { + command: `cat /opt/${appConfig.name}/config/version-history.txt`, + parser: 'text' + }, + failed: { + command: `cat /opt/${appConfig.name}/config/failed-versions.txt`, + parser: 'text' + } + }; + + const data = await api.getServerInfo( + Object.keys(api.expandServers(appConfig.servers)), + collector + ); + const result = { + latest: 0, + versions: [], + servers: [], + failed: [], + versionDates: new Map() + }; + + Object.values(data).forEach(entry => { + let serverVersions = []; + + entry.images.forEach(image => { + let version = parseInt(image.Tag, 10); + if (!Number.isNaN(version)) { + serverVersions.push(version); + } + + let date = new Date(image.CreatedAt); + let existingDate = result.versionDates.get(version); + if (!existingDate || existingDate.getTime() > date.getTime()) { + result.versionDates.set(version, date); + } + }); + + result.versions.push(...serverVersions); + + const serverFailed = entry.failed ? entry.failed.split('\n').map(v => parseInt(v, 10)) : []; + result.failed.push(...serverFailed); + + const history = entry.history ? entry.history.split('\n').map(v => parseInt(v, 10)) : []; + + result.servers.push({ + host: entry._host, + name: entry._serverName, + current: history[history.length - 1] || null, + previous: history[history.length - 2] || null, + versions: serverVersions.sort((a, b) => b - a), + history, + failed: serverFailed + }); + }); + + result.versions = Array.from(new Set(result.versions)).sort((a, b) => b - a); + result.failed = Array.from(new Set(result.failed)); + result.latest = result.versions[0] || 0; + result.current = mostCommon(result.servers.map(server => server.current)); + result.previous = mostCommon(result.servers.map(server => server.previous)); + + return result; } export function readFileFromTar(tarPath, filePath) { diff --git a/src/plugins/meteor/validate.js b/src/plugins/meteor/validate.js index 870e6ce2..13c524f1 100644 --- a/src/plugins/meteor/validate.js +++ b/src/plugins/meteor/validate.js @@ -151,5 +151,13 @@ export default function( ); } + if (config.app.enableUploadProgressBar) { + details = addDepreciation( + details, + 'enableUploadProgressBar', + 'This option is no longer used.' + ); + } + return addLocation(details, config.meteor ? 'meteor' : 'app'); } diff --git a/src/plugins/mongo/index.js b/src/plugins/mongo/index.js index 7aeac20e..0507bacf 100644 --- a/src/plugins/mongo/index.js +++ b/src/plugins/mongo/index.js @@ -47,3 +47,32 @@ export const hooks = { } } }; + +export async function checkSetup(api) { + const config = api.getConfig(); + if (!config.mongo) { + return []; + } + + const sessions = api.getSessions(['mongo']); + + return [ + { + sessions, + name: `mongo-${config.app.name}`, + setupKey: { + scripts: [ + api.resolvePath(__dirname, 'assets/mongo-setup.sh'), + api.resolvePath(__dirname, 'assets/templates/start.sh'), + api.resolvePath(__dirname, 'assets/mongo-start.sh') + ], + config: { + version: config.mongo.version + } + }, + containers: [ + 'mongodb' + ] + } + ]; +} diff --git a/src/plugins/proxy/assets/nginx.tmpl b/src/plugins/proxy/assets/nginx.tmpl index ba1a485b..6d9db078 100644 --- a/src/plugins/proxy/assets/nginx.tmpl +++ b/src/plugins/proxy/assets/nginx.tmpl @@ -62,7 +62,7 @@ gzip_types text/plain text/css application/javascript application/json applicati log_format vhost '$host $remote_addr - $remote_user [$time_local] ' '"$request" $status $body_bytes_sent ' - '"$http_referer" "$http_user_agent"'; + '"$http_referer" "$http_user_agent" $request_time $upstream_response_time $pipe'; access_log off; @@ -362,4 +362,4 @@ server { {{ end }} {{ end }} -{{ end }} \ No newline at end of file +{{ end }} diff --git a/src/plugins/proxy/assets/upstream.sh b/src/plugins/proxy/assets/upstream.sh index a7b8b3d0..d620d1fc 100644 --- a/src/plugins/proxy/assets/upstream.sh +++ b/src/plugins/proxy/assets/upstream.sh @@ -23,7 +23,7 @@ cat <<"EOT" > /opt/$PROXYNAME/upstream/$APPNAME ip_hash; <% } %> <% for(var index in hostnames) { %> -server <%= hostnames[index] %>:<%= port %>; +server <%= hostnames[index].host %>:<%= port %> <%= hostnames[index].params %>; <% } %> EOT diff --git a/src/plugins/proxy/command-handlers.js b/src/plugins/proxy/command-handlers.js index 24a3b6b3..fd179fc5 100644 --- a/src/plugins/proxy/command-handlers.js +++ b/src/plugins/proxy/command-handlers.js @@ -6,7 +6,7 @@ import fs from 'fs'; import nodemiral from '@zodern/nodemiral'; const log = debug('mup:module:proxy'); -const PROXY_CONTAINER_NAME = 'mup-nginx-proxy'; +export const PROXY_CONTAINER_NAME = 'mup-nginx-proxy'; export function logs(api) { log('exec => mup proxy logs'); @@ -47,7 +47,6 @@ export function leLogs(api) { export function setup(api) { log('exec => mup proxy setup'); const config = api.getConfig().proxy; - const serverConfig = api.getConfig().servers; const appConfig = api.getConfig().app; const appName = appConfig.name; @@ -145,8 +144,7 @@ export function setup(api) { } const hostnames = getLoadBalancingHosts( - serverConfig, - Object.keys(appConfig.servers) + api.expandServers(appConfig.servers), ); list.executeScript('Configure Nginx Upstream', { diff --git a/src/plugins/proxy/graceful-shutdown.js b/src/plugins/proxy/graceful-shutdown.js new file mode 100644 index 00000000..7b9b8745 --- /dev/null +++ b/src/plugins/proxy/graceful-shutdown.js @@ -0,0 +1,74 @@ +import { PROXY_CONTAINER_NAME } from './command-handlers'; +import { getLoadBalancingHosts, getSessions } from './utils'; + +function runScript(sessions, script, vars) { + const promises = sessions.map(session => + new Promise((resolve, reject) => { + session.executeScript(script, { + vars + }, (err, code, output) => { + console.log(err, code, output); + if (code > 0) { + return reject(output); + } + + resolve(); + }); + })); + + return Promise.all(promises); +} + +function updateUpstreams(api, toDrain) { + const { + app: appConfig, + proxy: config + } = api.getConfig(); + const hostnames = getLoadBalancingHosts( + api.expandServers(appConfig.servers), + toDrain ? [toDrain] : undefined + ); + const domains = config.domains.split(','); + + const proxySessions = getSessions(api); + + // TODO: we don't need to update the domains + return runScript( + proxySessions, + api.resolvePath(__dirname, 'assets/upstream.sh'), + { + domains, + name: appConfig.name, + setUpstream: !api.swarmEnabled() && config.loadBalancing, + stickySessions: config.stickySessions !== false, + proxyName: PROXY_CONTAINER_NAME, + port: appConfig.env.PORT, + hostnames + } + ); +} + +export async function gracefulShutdown(api, { session }) { + console.log('graceful shutdown'); + const { + proxy + } = api.getConfig(); + + if (!proxy || !proxy.loadBalancing) { + return; + } + + await updateUpstreams(api, session._host); +} + +export async function readdInstance(api) { + const { + proxy + } = api.getConfig(); + + if (!proxy || !proxy.loadBalancing) { + return; + } + + await updateUpstreams(api); +} diff --git a/src/plugins/proxy/index.js b/src/plugins/proxy/index.js index cf8d93d8..4e1f4bac 100644 --- a/src/plugins/proxy/index.js +++ b/src/plugins/proxy/index.js @@ -1,7 +1,8 @@ import * as _commands from './commands'; -import { addProxyEnv, normalizeUrl } from './utils'; -import { updateProxyForLoadBalancing } from './command-handlers'; +import { addProxyEnv, getLoadBalancingHosts, getSessions, normalizeUrl } from './utils'; +import { PROXY_CONTAINER_NAME, updateProxyForLoadBalancing } from './command-handlers'; import validator from './validate'; +import { gracefulShutdown, readdInstance } from './graceful-shutdown'; export const description = 'Setup and manage reverse proxy and ssl'; @@ -37,7 +38,7 @@ export function prepareConfig(config) { } config.app.env.HTTP_FORWARDED_COUNT = - config.app.env.HTTP_FORWARDED_COUNT || 1; + config.app.env.HTTP_FORWARDED_COUNT || 1; if (swarmEnabled) { config.app.docker.networks = config.app.docker.networks || []; @@ -125,7 +126,9 @@ export const hooks = { } }, 'post.reconfig': configureServiceHook, - 'post.proxy.setup': configureServiceHook + 'post.proxy.setup': configureServiceHook, + 'during.app.shutdown': gracefulShutdown, + 'during.app.start-instance': readdInstance }; export function swarmOptions(config) { @@ -135,3 +138,76 @@ export function swarmOptions(config) { }; } } + +export async function checkSetup(api) { + const config = api.getConfig(); + if (!config.proxy) { + return []; + } + + const sessions = getSessions(api); + + let configPaths = []; + + if (config.proxy.nginxServerConfig) { + configPaths.push( + api.resolvePath(api.getBasePath(), config.proxy.nginxServerConfig) + ); + } + if (config.proxy.nginxLocationConfig) { + configPaths.push( + api.resolvePath(api.getBasePath(), config.proxy.nginxLocationConfig) + ); + } + + if (config.proxy.ssl && config.proxy.ssl.crt) { + configPaths.push( + api.resolvePath(api.getBasePath(), config.proxy.ssl.crt) + ); + configPaths.push( + api.resolvePath(api.getBasePath(), config.proxy.ssl.key) + ); + } + + let upstream = []; + + if (config.loadBalancing) { + upstream = getLoadBalancingHosts( + api.expandServers(config.app.servers) + ); + } + + return [ + { + sessions, + name: `proxy-${config.app.name}`, + setupKey: { + // TODO: handle legacy ssl configuration + scripts: [ + api.resolvePath(__dirname, 'assets/proxy-setup.sh'), + api.resolvePath(__dirname, 'assets/templates/start.sh'), + api.resolvePath(__dirname, 'assets/nginx.tmpl'), + api.resolvePath(__dirname, 'assets/nginx-config.sh'), + api.resolvePath(__dirname, 'assets/ssl-cleanup.sh'), + api.resolvePath(__dirname, 'assets/ssl-setup.sh'), + api.resolvePath(__dirname, 'assets/upstream.sh'), + api.resolvePath(__dirname, 'assets/proxy-start.sh') + ], + config: { + domains: config.proxy.domains, + app: config.app.name, + letsEncryptEmail: config.ssl ? config.ssl.letsEncryptEmail : '', + swarm: api.swarmEnabled(), + clientUploadLimit: config.clientUploadLimit, + upstream, + stickySessions: config.stickySessions, + appPort: config.app.env.PORT + } + }, + containers: [ + `${PROXY_CONTAINER_NAME}-letsencrypt`, + PROXY_CONTAINER_NAME + ] + } + ]; +} diff --git a/src/plugins/proxy/utils.js b/src/plugins/proxy/utils.js index 9ade7a99..6fde4d71 100644 --- a/src/plugins/proxy/utils.js +++ b/src/plugins/proxy/utils.js @@ -44,8 +44,10 @@ export function normalizeUrl(config, env) { return _config.app.env.ROOT_URL; } -export function getLoadBalancingHosts(serverConfig, serverNames) { - return serverNames.map(name => - serverConfig[name].privateIp || serverConfig[name].host +export function getLoadBalancingHosts(expandedServers, drainingHosts = []) { + return Object.values(expandedServers).map(({ server }) => ({ + host: server.privateIp || server.host, + params: drainingHosts.includes(server.host) ? 'down' : '' + }) ); } diff --git a/src/server-info.js b/src/server-info.js index dbf904e1..b4a86b79 100644 --- a/src/server-info.js +++ b/src/server-info.js @@ -42,7 +42,14 @@ export const builtInParsers = { return null; }, - jsonArray: parseJSONArray + jsonArray: parseJSONArray, + text(stdout, code) { + if (code === 0) { + return stdout.trim(); + } + + return null; + } }; export const _collectors = { @@ -78,8 +85,9 @@ function generateVarCommand(name, command) { return ` echo "${prefix}${name}${suffix}" ${command} 2>&1 + MUP_CODE=$? echo "${codeSeperator}" - echo $? + echo $MUP_CODE `; } diff --git a/src/server-sources.js b/src/server-sources.js new file mode 100644 index 00000000..8d624b3a --- /dev/null +++ b/src/server-sources.js @@ -0,0 +1,9 @@ +export const serverSources = Object.create(null); + +export function registerServerSource(type, { load, upToDate, update } = {}) { + if (type in serverSources) { + throw new Error(`Duplicate server sources: ${type}`); + } + + serverSources[type] = { load, upToDate, update }; +} diff --git a/src/tasks/assets/check-setup.sh b/src/tasks/assets/check-setup.sh new file mode 100644 index 00000000..c1e81874 --- /dev/null +++ b/src/tasks/assets/check-setup.sh @@ -0,0 +1,18 @@ +set -e + +<% for(const [ name, value ] of Object.entries(keyHashes)) { %> +if [ $(sudo cat /opt/.mup-setup/<%- name %>.txt) != "<%- value %>" ]; then + exit 1 +fi +<% } %> + +<% for(const service of services) { %> +sudo service <%- service %> status +<% } %> + +<% for (const container of containers) { %> +STATUS="$(sudo docker inspect --format='{{.State.Running}}' <%- container %> 2> /dev/null)" +if [ "$STATUS" == 'false' ] || [ -z "$STATUS" ]; then + exit 1 +fi +<% } %> diff --git a/src/utils.js b/src/utils.js index e04f71f2..f5c66701 100644 --- a/src/utils.js +++ b/src/utils.js @@ -33,11 +33,21 @@ export function runTaskList(list, sessions, opts) { delete opts.verbose; } - if (opts && opts.showDuration) { + if (opts) { + let pluginApi = opts._mupPluginApi; list._taskQueue.forEach(task => { task.options = task.options || {}; - task.options.showDuration = true; + + if (task.type === '_runHook') { + task.options._getMupApi = () => () => pluginApi; + } + + if (opts.showDuration) { + task.options.showDuration = true; + } }); + + delete opts._mupPluginApi; delete opts.showDuration; } @@ -240,7 +250,7 @@ export function forwardPort({ remotePort, onReady, onError, - onConnection = () => {} + onConnection = () => { } }) { const sshOptions = createSSHOptions(server); const netServer = net.createServer(netConnection => { diff --git a/src/validate/utils.js b/src/validate/utils.js index 7a056b97..5250b459 100644 --- a/src/validate/utils.js +++ b/src/validate/utils.js @@ -66,10 +66,12 @@ export function serversExist(serversConfig = {}, serversUsed = {}) { } export function addDepreciation(details, path, reason, link) { + const learnMore = link ? `\n Learn more at ${link}` : ''; + details.push({ type: 'depreciation', path, - message: `${reason}\n Learn more at ${link}` + message: `${reason}${learnMore}` }); return details;