diff --git a/clouder_template_elasticsearch/README.rst b/clouder_template_elasticsearch/README.rst new file mode 100644 index 0000000..4ab4931 --- /dev/null +++ b/clouder_template_elasticsearch/README.rst @@ -0,0 +1,52 @@ +.. image:: https://img.shields.io/badge/licence-LGPL--3-blue.svg + :target: http://www.gnu.org/licenses/lgpl-3.0-standalone.html + :alt: License: LGPL-3 + +================================ +Clouder Template - Elasticsearch +================================ + +This module provides a template for Elasticsearch in Clouder. + + +Configuration +============= + +Clouder configuration instructions are available at https://clouder.readthedocs.io/ + +Usage +===== + +To use this module, you need to: + +#. Create an Elasticsearch Service in the Clouder Control Panel + +Known issues / Roadmap +====================== + +* Add ElasticDump db backup & restore +* ElasticDump should be a separate service +* Add SSL (Requires CA - https://github.com/clouder-community/clouder/issues/167) + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues +`_. In case of trouble, please +check there if your issue has already been reported. If you spotted it first, +help us smashing it by providing a detailed and welcomed feedback. + +Credits +======= + +Contributors +------------ + +* Dave Lasley + +Maintainer +---------- + +This module is maintained by Clouder Community. + +To contribute to this module, please visit https://github.com/clouder-community/clouder diff --git a/clouder_template_elasticsearch/__init__.py b/clouder_template_elasticsearch/__init__.py new file mode 100644 index 0000000..6b7b00c --- /dev/null +++ b/clouder_template_elasticsearch/__init__.py @@ -0,0 +1,5 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 LasLabs Inc. +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl.html). + +from . import models diff --git a/clouder_template_elasticsearch/__manifest__.py b/clouder_template_elasticsearch/__manifest__.py new file mode 100644 index 0000000..d79de5d --- /dev/null +++ b/clouder_template_elasticsearch/__manifest__.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 LasLabs Inc. +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl.html). + +{ + 'name': 'Clouder Template Elasticsearch', + 'version': '10.0.10.0.0', + 'category': 'Clouder', + 'depends': [ + 'clouder', + 'clouder_template_proxy', + 'clouder_template_dns', + ], + 'author': 'LasLabs Inc.', + 'license': 'LGPL-3', + 'website': 'https://github.com/clouder-community/clouder', + 'data': [ + 'data/image_template.xml', + 'data/image.xml', + 'data/image_port.xml', + 'data/image_volume.xml', + 'data/application_type.xml', + 'data/application_template.xml', + 'data/application.xml', + 'data/application_link.xml', + ], + 'installable': True, + 'application': False, +} diff --git a/clouder_template_elasticsearch/data/application.xml b/clouder_template_elasticsearch/data/application.xml new file mode 100644 index 0000000..c3e25fe --- /dev/null +++ b/clouder_template_elasticsearch/data/application.xml @@ -0,0 +1,44 @@ + + + + + + + Elasticsearch Data + data + + + + 1 + + + + + Elasticsearch Exec + exec + + + + 2 + + auto + + + + Elasticsearch + elasticsearch + + + + 1 + + + + diff --git a/clouder_template_elasticsearch/data/application_link.xml b/clouder_template_elasticsearch/data/application_link.xml new file mode 100644 index 0000000..73bf2ef --- /dev/null +++ b/clouder_template_elasticsearch/data/application_link.xml @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + diff --git a/clouder_template_elasticsearch/data/application_template.xml b/clouder_template_elasticsearch/data/application_template.xml new file mode 100644 index 0000000..a13d396 --- /dev/null +++ b/clouder_template_elasticsearch/data/application_template.xml @@ -0,0 +1,13 @@ + + + + + + + Elasticsearch + + + diff --git a/clouder_template_elasticsearch/data/application_type.xml b/clouder_template_elasticsearch/data/application_type.xml new file mode 100644 index 0000000..ea8d64e --- /dev/null +++ b/clouder_template_elasticsearch/data/application_type.xml @@ -0,0 +1,14 @@ + + + + + + + elasticsearch + elasticsearch + + + diff --git a/clouder_template_elasticsearch/data/image.xml b/clouder_template_elasticsearch/data/image.xml new file mode 100644 index 0000000..e4ad392 --- /dev/null +++ b/clouder_template_elasticsearch/data/image.xml @@ -0,0 +1,24 @@ + + + + + + + image_elasticsearch_data + + lasley/elasticsearch-data + + + + image_elasticsearch_exec + + lasley/elasticsearch-exec + data + + + diff --git a/clouder_template_elasticsearch/data/image_port.xml b/clouder_template_elasticsearch/data/image_port.xml new file mode 100644 index 0000000..64a28eb --- /dev/null +++ b/clouder_template_elasticsearch/data/image_port.xml @@ -0,0 +1,23 @@ + + + + + + + + http + 9200 + + + + + transport + 9300 + + + diff --git a/clouder_template_elasticsearch/data/image_template.xml b/clouder_template_elasticsearch/data/image_template.xml new file mode 100644 index 0000000..ec1308a --- /dev/null +++ b/clouder_template_elasticsearch/data/image_template.xml @@ -0,0 +1,19 @@ + + + + + + + image_template_elasticsearch_data + + + + image_template_elasticsearch_exec + + + diff --git a/clouder_template_elasticsearch/data/image_volume.xml b/clouder_template_elasticsearch/data/image_volume.xml new file mode 100644 index 0000000..b64e60e --- /dev/null +++ b/clouder_template_elasticsearch/data/image_volume.xml @@ -0,0 +1,61 @@ + + + + + + + + data + /opt/elasticsearch/data + elasticsearch + + + + + etc + /opt/elasticsearch/config + elasticsearch + + + + + log + /opt/elasticsearch/logs + elasticsearch + + + + + lib + /opt/elasticsearch/lib + elasticsearch + + + + + module + /opt/elasticsearch/modules + elasticsearch + + + + + plugin + /opt/elasticsearch/plugins + elasticsearch + + + diff --git a/clouder_template_elasticsearch/images/elasticsearch5-data/Dockerfile b/clouder_template_elasticsearch/images/elasticsearch5-data/Dockerfile new file mode 100644 index 0000000..630b028 --- /dev/null +++ b/clouder_template_elasticsearch/images/elasticsearch5-data/Dockerfile @@ -0,0 +1,6 @@ +FROM clouder/base:3.4 +MAINTAINER Dave Lasley + +COPY etc ./config + +CMD tail -f /dev/null diff --git a/clouder_template_elasticsearch/images/elasticsearch5-data/etc/elasticsearch.yml b/clouder_template_elasticsearch/images/elasticsearch5-data/etc/elasticsearch.yml new file mode 100644 index 0000000..bee2cfe --- /dev/null +++ b/clouder_template_elasticsearch/images/elasticsearch5-data/etc/elasticsearch.yml @@ -0,0 +1,5 @@ +network.host: 0.0.0.0 + +# this value is required because we set "network.host" +# be sure to modify it appropriately for a production cluster deployment +discovery.zen.minimum_master_nodes: 1 diff --git a/clouder_template_elasticsearch/images/elasticsearch5-data/etc/logging.yml b/clouder_template_elasticsearch/images/elasticsearch5-data/etc/logging.yml new file mode 100644 index 0000000..c2681ac --- /dev/null +++ b/clouder_template_elasticsearch/images/elasticsearch5-data/etc/logging.yml @@ -0,0 +1,15 @@ +# you can override this using by setting a system property, for example -Des.logger.level=DEBUG +es.logger.level: INFO +rootLogger: ${es.logger.level}, console +logger: + # log action execution errors for easier debugging + action: DEBUG + # reduce the logging for aws, too much is logged under the default INFO + com.amazonaws: WARN + +appender: + console: + type: console + layout: + type: consolePattern + conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n" diff --git a/clouder_template_elasticsearch/images/elasticsearch5-exec/Dockerfile b/clouder_template_elasticsearch/images/elasticsearch5-exec/Dockerfile new file mode 100644 index 0000000..fdb9aed --- /dev/null +++ b/clouder_template_elasticsearch/images/elasticsearch5-exec/Dockerfile @@ -0,0 +1,65 @@ +FROM clouder/base:3.4 +MAINTAINER Dave Lasley + +# Loosely based on https://github.com/docker-library/elasticsearch/blob/master/Dockerfile.template + +ENV ES_VERSION 5.0.1 +ENV ES_BASE https://artifacts.elastic.co/downloads/elasticsearch + +ENV GOSU_VERSION 1.10 +ENV GOSU_BASE https://github.com/tianon/gosu/releases/download + +ENV PACKAGES "ca-certificates curl nodejs openjdk8-jre openssl wget" + +RUN apk add --update $PACKAGES + +WORKDIR /tmp + +# Install glibc +RUN apk --no-cache add ca-certificates openssl \ + && wget -q -O /etc/apk/keys/sgerrand.rsa.pub https://raw.githubusercontent.com/sgerrand/alpine-pkg-glibc/master/sgerrand.rsa.pub \ + && apk --no-cache -X http://apkproxy.heroku.com/sgerrand/alpine-pkg-glibc add glibc glibc-bin + +# Install Go based sudo (gosu) +RUN set -x \ + && apk add gnupg \ + && wget -O /usr/local/bin/gosu "$GOSU_BASE/$GOSU_VERSION/gosu-$(apk --print-arch |sed -e 's/x86_64/amd64/')" \ + && wget -O /usr/local/bin/gosu.asc "$GOSU_BASE/$GOSU_VERSION/gosu-$(apk --print-arch |sed -e 's/x86_64/amd64/').asc" \ + && export GNUPGHOME="$(mktemp -d)" \ + && gpg2 --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 \ + && gpg2 --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu \ + && rm -r "$GNUPGHOME" /usr/local/bin/gosu.asc \ + && chmod +x /usr/local/bin/gosu \ + && gosu nobody true \ + && apk del gnupg + +# Install Elasticsearch +WORKDIR /opt + +RUN mkdir -p /opt \ + && adduser -h /opt/elasticsearch -g elasticsearch -s /bin/bash -D elasticsearch + +RUN ln -s elasticsearch elasticsearch-$ES_VERSION +USER elasticsearch +RUN set -x \ + && wget -O - "${ES_BASE}/elasticsearch-${ES_VERSION}.tar.gz" | tar -xz + +ENV PATH /opt/elasticsearch/bin:$PATH + +WORKDIR /opt/elasticsearch +RUN set -ex \ + && for path in \ + ./data \ + ./logs \ + ./config \ + ./config/scripts \ + ; do \ + mkdir -p "$path"; \ + chown -R elasticsearch:elasticsearch "$path"; \ + done + +# Entrypoint +COPY docker-entrypoint.sh / + +ENTRYPOINT ["/docker-entrypoint.sh"] +CMD ["elasticsearch"] diff --git a/clouder_template_elasticsearch/images/elasticsearch5-exec/docker-entrypoint.sh b/clouder_template_elasticsearch/images/elasticsearch5-exec/docker-entrypoint.sh new file mode 100755 index 0000000..39ba938 --- /dev/null +++ b/clouder_template_elasticsearch/images/elasticsearch5-exec/docker-entrypoint.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +# ElasticSearch Docker Entrypoint: +# https://github.com/docker-library/elasticsearch +# Apache Licensed: +# https://github.com/docker-library/elasticsearch/blob/b6fe7dbafc4508c3a1fcf40cbe74c06188c361aa/LICENSE + +set -e + +# Add elasticsearch as command if needed +if [ "${1:0:1}" = '-' ]; then + set -- elasticsearch "$@" +fi + +# Drop root privileges if we are running elasticsearch +# allow the container to be started with `--user` +if [ "$1" = 'elasticsearch' -a "$(id -u)" = '0' ]; then + # Change the ownership of /usr/share/elasticsearch/data to elasticsearch + chown -R elasticsearch:elasticsearch /usr/share/elasticsearch/data + + set -- gosu elasticsearch "$@" + #exec gosu elasticsearch "$BASH_SOURCE" "$@" +fi + +# As argument is not related to elasticsearch, +# then assume that user wants to run his own process, +# for example a `bash` shell to explore this image +exec "$@" diff --git a/clouder_template_elasticsearch/models/__init__.py b/clouder_template_elasticsearch/models/__init__.py new file mode 100644 index 0000000..a3df682 --- /dev/null +++ b/clouder_template_elasticsearch/models/__init__.py @@ -0,0 +1,5 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 LasLabs Inc. +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl.html). + +from . import backup diff --git a/clouder_template_elasticsearch/models/backup copy.py b/clouder_template_elasticsearch/models/backup copy.py new file mode 100644 index 0000000..612fedb --- /dev/null +++ b/clouder_template_elasticsearch/models/backup copy.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 LasLabs Inc. +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl.html). + +from odoo import api, models + + +class ClouderBackup(models.Model): + """ It provides Elasticsearch context for Clouder Saves + + All methods and properties are to be prefixed with ``elastic_`` in order + to prevent namespace clashes with existing operations, unless overloading + and calling + returning the super. + """ + + _inherit = 'clouder.backup' + + @api.multi + def backup_database(self): + if self.base_id.service_id.db_type == 'elasticsearch': + pass + return super(ClouderBackup, self).backup_database() + + @api.multi + def restore_database(self): + if self.base_id.service_id.db_type == 'elasticsearch': + pass + return super(ClouderBackup, self).restore_database() diff --git a/clouder_template_elasticsearch/models/backup.py b/clouder_template_elasticsearch/models/backup.py new file mode 100644 index 0000000..c18a0a0 --- /dev/null +++ b/clouder_template_elasticsearch/models/backup.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 LasLabs Inc. +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl.html). + +from os import path + +from odoo import api, models + + +class ClouderBackup(models.Model): + """ It provides Elasticsearch context for Clouder Backups + + All public methods and properties are to be prefixed with ``elastic_`` in + order to prevent namespace clashes with existing operations, unless + overloading and calling + returning the super. + """ + + _inherit = 'clouder.backup' + + @property + def elasticsearch_volumes_data(self): + """ It returns a ``clouder.image.volume`` Recordset for data vols. """ + res = self.env['clouder.image.volume'] + for volume in ['data', 'etc', 'log']: + res += self.__get_internal_ref( + 'image_volume_elasticsearch_%s' % volume, + ) + return res + + @api.multi + def deploy_base(self): + """ It backs up the file store """ + res = super(ClouderBackup, self).deploy_base() + if self.base_id.application_id.type_id.name == 'elasticsearch': + self.elasticsearch_deploy_base() + return res + + @api.multi + def restore_base(self): + """ It restores a backup of the file store """ + res = super(ClouderBackup, self).deploy_base() + if self.base_id.application_id.type_id.name == 'elasticsearch': + self.elasticsearch_restore_base() + return res + + @api.multi + def elasticsearch_deploy_base(self): + """ It backs up the filestore for Elasticsearch applications. """ + for volume in self.elasticsearch_volumes_data(): + self.service_id.base_backup_container.execute([ + 'rsync', '--progres', '-aze', + volume.localpath, + path.join(self.BACKUP_BASE_DIR, self.name, volume.name), + ], + username=self.base_id.application_id.type_id.system_user, + ) + + @api.multi + def elasticsearch_restore_base(self, base): + """ It restores a backup of the file store """ + for volume in self.elasticsearch_volumes_data(): + self.service_id.base_backup_container.execute([ + 'rsync', '--progres', '-aze', '--delete', + path.join(self.BACKUP_BASE_DIR, self.name, volume.name), + volume.localpath, + ], + username=self.base_id.application_id.type_id.system_user, + ) + + def __get_internal_ref(self, name): + return self.env.ref('clouder_template_elasticsearch.%s' % name)