diff --git a/.metadata b/.metadata new file mode 100644 index 0000000..fdd1b78 --- /dev/null +++ b/.metadata @@ -0,0 +1 @@ +language_type: cloudformation diff --git a/.taskcat.yml b/.taskcat.yml index 6bb3790..d41a55e 100644 --- a/.taskcat.yml +++ b/.taskcat.yml @@ -3,36 +3,38 @@ project: owner: quickstart@amazon.com package_lambda: false regions: - - ap-northeast-1 - - ap-northeast-2 - - ap-south-1 - - ap-southeast-1 - - ap-southeast-2 - - ca-central-1 - - eu-central-1 - - eu-west-1 - - eu-west-2 - - sa-east-1 - - us-east-1 - - us-east-2 - - us-west-1 - us-west-2 s3_bucket: '' tests: splunk-enterprise: parameters: AvailabilityZones: $[taskcat_genaz_2] - HECClientLocation: 10.0.0.0/16 - KeyName: taskcat-test + KeyName: [your-aws-key] QSS3BucketName: $[taskcat_autobucket] + QSS3KeyPrefix: quickstart-splunk-enterprise/ QSS3BucketRegion: $[taskcat_current_region] - SSHClientLocation: 10.0.0.0/16 - SplunkAdminPassword: $[taskcat_genpass_10] - SplunkClusterSecret: $[taskcat_genpass_10] - SplunkIndexerDiscoverySecret: $[taskcat_genpass_10] - WebClientLocation: 72.21.196.66/32 - regions: - - us-west-1 - - us-east-2 + SplunkAdminPassword: $[taskcat_genpass_8S] + SplunkClusterSecret: $[taskcat_genpass_8S] + SplunkIndexerDiscoverySecret: $[taskcat_genpass_8S] + IndexerInstanceType: i3.4xlarge + SearchHeadInstanceType: c5.4xlarge + SplunkIndexerDiskSize: '300' + SplunkSearchHeadDiskSize: '300' + SHCEnabled: 'no' + SplunkIndexerCount: 4 + SplunkReplicationFactor: 2 + SplunkSearchFactor: 2 + SplunkLicenseBucket: [splunk-license-bucket-name] + SplunkLicensePath: [splunk-license-file] + WebClientLocation: 1.2.3.4/32 + HECClientLocation: 1.2.3.4/32 + SSHClientLocation: 1.2.3.4/32 + VPCCIDR: 10.0.0.0/16 + PublicSubnet1CIDR: 10.0.1.0/24 + PublicSubnet2CIDR: 10.0.2.0/24 + PublicSubnet3CIDR: 10.0.3.0/24 + NumberOfAZs: 2 + SmartStoreBucketName: $[taskcat_autobucket] + regions: $[taskcat_current_region] s3_bucket: '' - template: templates/splunk-enterprise-master.template.yaml + template: templates/splunk-enterprise-main.template.yaml diff --git a/README.md b/README.md index 955e8f8..b8c6536 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ View the accompanying [deployment guide](https://fwd.aws/bGBmy) for everything y ### Prerequisites -Before getting started with the template configuration, you will need to make your Splunk Enterprise license privately accessible for CloudFormation template deployment via S3 download. The following steps will guide you through that process. *(Note: This step is not required, and you can upload your license from the Splunk web interface. It is, however, required that you have a non-trial Splunk Enterprise license to fully utilize the deployment our template creates. If you don't already have a Splunk Enterprise license, you can obtain one by contacting sales@splunk.com.)* +Before getting started with the template configuration, you will need to make your Splunk Enterprise license privately accessible for CloudFormation template deployment via S3 download. The following steps will guide you through that process. *(Note: This step is required. A non-trial Splunk Enterprise license is required to allow our template to configure the Splunk deployment. If you don't already have a Splunk Enterprise license, you can obtain one by contacting sales@splunk.com.)* 1. From the AWS Console, select "S3" under the "Storage" heading, or by simply typing "S3" into the search bar. 2. You can either select an existing private bucket to upload to, or create a new one. If you select an existing bucket, make sure its access policy does not grant public access. By default, all the S3 resources are private, so only the AWS account that created the resources can access them. For this exercise, I'm outlining how to create a new bucket. diff --git a/scripts/user_data.sh b/scripts/user_data.sh new file mode 100644 index 0000000..aa2b69c --- /dev/null +++ b/scripts/user_data.sh @@ -0,0 +1,614 @@ +#!/bin/bash -xe + +#### start universal functions +function base +{ + + # variables + export LOCALIP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4) + export INSTANCEID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id) + export SPLUNK_USER=splunk + export SPLUNK_BIN=/opt/splunk/bin/splunk + export SPLUNK_HOME=/opt/splunk + + # make cloud-init output log readable by root only to protect sensitive parameter values + chmod 600 /var/log/cloud-init-output.log + + #- Newer versions of the Splunk AMI do not come with Splunk pre-installed. Instead Splunk + #- is installed via ansible as part of cloud-init. The following code (line 23 & 24) is + #- needed to ensure the ansible code is ran prior to the remainder of this script. Without + #- explicitly executing ansible first, the configuration in this user_data.sh script tries + #- (and fails) to execute as there isn't a Splunk deployment to configure. + + # run the ansible code + (cd /opt/splunk-ansible && time sudo -u ec2-user -E -S bash -c "SPLUNK_BUILD_URL=/tmp/splunk.tgz SPLUNK_ENABLE_SERVICE=true SPLUNK_PASSWORD=SPLUNK-$(wget -q -O - http://169.254.169.254/latest/meta-data/instance-id) ansible-playbook -i inventory/environ.py site.yml") + + # setup auth with user-selected admin password + mv $SPLUNK_HOME/etc/passwd $SPLUNK_HOME/etc/passwd.bak + cat >> $SPLUNK_HOME/etc/system/local/user-seed.conf << end + [user_info] + USERNAME = admin + PASSWORD = $ADMIN_PASSWORD +end + + sed -i '/guid/d' $SPLUNK_HOME/etc/instance.cfg + touch $SPLUNK_HOME/etc/.ui_login + + # restart Splunk for admin password update + $SPLUNK_BIN restart +} + +function restart_signal +{ + + # restart splunk + $SPLUNK_BIN restart + + # communicate back to CloudFormation the status of the instance creation + /opt/aws/bin/cfn-signal -e $? --stack $STACK_NAME --resource $RESOURCE --region $AWS_REGION + + # disable splunk user login + usermod --expiredate 1 splunk +} + +#### end universal config + +##### +#### start role-specific functions +##### + +### +# setup nvme drives for i3 indexers +function nvme_setup +{ + # first, determine the instance type. + ec2_type=$(curl -s http://169.254.169.254/latest/meta-data/instance-type) + + # this script is intended to run on i3* instance types. + if [[ "$ec2_type" != *"i3"* ]] + then + return 0 + fi + + # find the attached nvme drives. lsblk could work here, but utilizing the nvme-list utility due to + # json formatting and simpler parsing. install the nvme-cli and jq packages to accomplish this. + yum -y install nvme-cli jq >/dev/null + + # save the nvme drive information to a temp file for parsing + nvme list --output-format=json > /tmp/nvme_drive.json + + # declare the nvme device array + declare -a nvme_devices + unset nvme_devices + + for nvme_device in $(jq '.Devices[] | .DevicePath' /tmp/nvme_drive.json) + do + # test to ensure that the storage device is instance storage. in testing, I have + # seen EBS volues show as NVME. this logic will ensure attached EBS devices are not + # added to the nvme raid0 + nvme_model_type=$(jq -r '.Devices[] | select(.DevicePath=='$nvme_device') | .ModelNumber' /tmp/nvme_drive.json) + if [[ $nvme_model_type = *"NVMe Instance Storage"* ]] + then + # unfortunate 'hack' here to remove the quotes from the device name. without them, the jq lookup + # will fail in the previous step. however, they need to be removed for the md raid creation later. + # additionally, since there needs to be a space between device names for the md create, convert + # quotes to spaces, and remove leading space. this leaves "$nvme_device " (note trailing space) + # stored in the array. this will allow for simply using the contents of the array as an argument for + # building the raid0 device + nvme_device=$(echo $nvme_device|sed 's/"/ /g'| sed 's/^ //g') + + # save device list in nvme_devices array + nvme_devices+=("$nvme_device") + else + # if the nvme model type is not instance storage, continue to the next iteration of the loop + continue + fi + done + + # name of the raid device to create + raid_device="/dev/md0" + + # mount point of the raid device + raid_mount="/opt/splunk" + + # make directory for mount point + mkdir -p $raid_mount + + # create the raid device + mdadm --create $raid_device --level=raid0 --raid-devices=${#nvme_devices[@]} ${nvme_devices[@]} + + # create filesystem on raid device + if [ ${#nvme_devices[@]} -eq 1 ] + then + discardOption="" + else + discardOption="-E nodiscard" + fi + + mkfs.ext4 -m 2 -F -F ${discardOption} $raid_device + + # add entry to fstab for mounting on reboot + echo "$raid_device $raid_mount auto defaults,nofail,noatime 0 2" >>/etc/fstab + + # mount device + mount $raid_device + +} + +### +# Splunk Cluster Manager / License Manager +### +function splunk_cm +{ + # execute base install and configuration + base + + #export RESOURCE="SplunkCM" + printf '%s\t%s\n' "$LOCALIP" 'splunklicense' >> /etc/hosts + hostname splunklicense + + #- for the CM, we can't reference CM_PRIVATEIP in the CloudFormation UserData like + #- we do in the other resources because the CM hasn't been created yet. To keep the + #- syntax consistent across each resource in user_data.sh, export $CM_PRIVATEIP to + #- the CM's local ip address + export CM_PRIVATEIP=$LOCALIP + + # Install license from metadata. + if [ $INSTALL_LICENSE = 1 ]; then + mkdir -p $SPLUNK_HOME/etc/licenses/enterprise/ + mv /tmp/splunk.license $SPLUNK_HOME/etc/licenses/enterprise/splunk.license + chown -R $SPLUNK_USER:$SPLUNK_USER $SPLUNK_HOME/etc/licenses/enterprise + #/opt/aws/bin/cfn-init -v --stack $STACK_NAME --resource $RESOURCE --region $AWS_REGION + fi + + # Increase splunkweb connection timeout with splunkd + mkdir -p $SPLUNK_HOME/etc/apps/base-autogenerated/local + cat >>$SPLUNK_HOME/etc/apps/base-autogenerated/local/web.conf <>$SPLUNK_HOME/etc/apps/base-autogenerated/local/outputs.conf <>$SPLUNK_HOME/etc/system/local/server.conf < /tmp/token + TOKEN=`sed -n 's/\\ttoken=//p' /tmp/token` && rm /tmp/token + + # place generated config into master-apps + mkdir -p $SPLUNK_HOME/etc/master-apps/peer-base-autogenerated/local + mv $SPLUNK_HOME/etc/apps/splunk_httpinput/local/inputs.conf $SPLUNK_HOME/etc/master-apps/peer-base-autogenerated/local + + # peer config 2: enable splunk tcp input + cat >>$SPLUNK_HOME/etc/master-apps/peer-base-autogenerated/local/inputs.conf <>$SPLUNK_HOME/etc/master-apps/_cluster/local/indexes.conf <> $SPLUNK_HOME/etc/slave-apps/_cluster/local/indexes.conf << end + [default] + repFactor = auto + remotePath = volume:remote_store/splunk_db/$_index_name + coldPath=$SPLUNK_DB/$_index_name/colddb + thawedPath=$SPLUNK_DB/$_index_name/thaweddb +end + + cat >>$SPLUNK_HOME/etc/slave-apps/_cluster/local/indexes.conf <>$SPLUNK_HOME/etc/apps/base-autogenerated/local/web.conf <>$SPLUNK_HOME/etc/apps/base-autogenerated/local/server.conf <>$SPLUNK_HOME/etc/system/local/server.conf <> /etc/hosts + hostname "splunksearch-$num" + + # set splunk servername + sudo -u $SPLUNK_USER $SPLUNK_BIN set servername SHC$num + + # Increase splunkweb connection timeout with splunkd + cat >$SPLUNK_HOME/etc/system/local/web.conf <>$SPLUNK_HOME/etc/system/local/server.conf <> /etc/hosts + hostname splunk-shc-deployer + + # Increase splunkweb connection timeout with splunkd + mkdir -p $SPLUNK_HOME/etc/apps/base-autogenerated/local + cat >>$SPLUNK_HOME/etc/apps/base-autogenerated/local/web.conf <>$SPLUNK_HOME/etc/apps/base-autogenerated/local/server.conf <>$SPLUNK_HOME/etc/apps/base-autogenerated/local/outputs.conf <>$SPLUNK_HOME/etc/shcluster/apps/member-base-autogenerated/local/outputs.conf <> /etc/hosts + hostname splunksearch + + # Increase splunkweb connection timeout with splunkd + mkdir -p $SPLUNK_HOME/etc/apps/base-autogenerated/local + cat >>$SPLUNK_HOME/etc/apps/base-autogenerated/local/web.conf <>$SPLUNK_HOME/etc/apps/base-autogenerated/local/outputs.conf < NumberOfAZs: AllowedValues: - '2' - '3' Default: '2' - Description: Number of Availability Zones to use in the VPC. This must match your - selections in the list of Availability Zones parameter. + Description: Number of Availability Zones to use in the VPC. This must match your selections in the list of Availability Zones parameter. Type: String WebClientLocation: + Default: '0.0.0.0/0' AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$ - ConstraintDescription: Must be a valid IP range in x.x.x.x/x notation. Use 0.0.0.0/0 - for no restrictions. - Description: 'The IP address range that is allowed to connect to the Splunk web - interface. Note: a value of 0.0.0.0/0 will allow access from ANY ip address' + ConstraintDescription: Must be a valid IP range in x.x.x.x/x notation. Use 0.0.0.0/0 for no restrictions. + Description: The IP address range that is allowed to connect to the Splunk web interface. Note that a value of 0.0.0.0/0 will allow access from ANY ip address. MaxLength: '19' MinLength: '9' Type: String HECClientLocation: + Default: '0.0.0.0/0' AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$ - ConstraintDescription: Must be a valid IP range in x.x.x.x/x notation. Use 0.0.0.0/0 - for no restrictions. - Description: 'The IP address range that is allowed to send data to Splunk HTTP - Event Collector. Note: a value of 0.0.0.0/0 will allow access from ANY ip address' + ConstraintDescription: Must be a valid IP range in x.x.x.x/x notation. Use 0.0.0.0/0 for no restrictions. + Description: 'The IP address range that is allowed to send data to Splunk HTTP Event Collector. Note that a value of 0.0.0.0/0 will allow access from ANY ip address.' MaxLength: '19' MinLength: '9' Type: String IndexerInstanceType: AllowedValues: - - c4.2xlarge - - c4.4xlarge - - c4.8xlarge - - m4.2xlarge - - m4.4xlarge - - m4.10xlarge - - c5.2xlarge + - m5.4xlarge + - m5.8xlarge - c5.4xlarge - c5.9xlarge - c5.18xlarge - - i3.2xlarge - i3.4xlarge - i3.8xlarge + - i3.16xlarge + - i3en.3xlarge + - i3en.6xlarge + - i3en.12xlarge + - i3en.24xlarge Description: EC2 instance type for Splunk Indexers ConstraintDescription: must be a valid EC2 instance type. - Default: c5.4xlarge + Default: i3.4xlarge Type: String SearchHeadInstanceType: AllowedValues: - - c4.2xlarge - - c4.4xlarge - - c4.8xlarge - - r4.4xlarge - - r4.8xlarge - - r4.16xlarge - - c5.2xlarge + - r5.4xlarge + - r5.8xlarge + - r5.16xlarge - c5.4xlarge - c5.9xlarge + - c5.18xlarge - m5.2xlarge - m5.4xlarge + - m5.8xlarge - m5.12xlarge Description: EC2 instance type for Splunk Search Heads ConstraintDescription: must be a valid EC2 instance type. Default: c5.4xlarge Type: String - IndexerApps: - Description: Comma separated list of URLs of Splunk App (or Add-on) tarballs (.spl) - to pre-install on indexer(s) - Default: '' - Type: CommaDelimitedList - SearchHeadApps: - Description: Comma separated list of URLs of Splunk App (or Add-on) tarballs (.spl) - to pre-install on search head(s) - Default: '' - Type: CommaDelimitedList KeyName: ConstraintDescription: Must be the name of an existing EC2 KeyPair. Description: Name of an existing EC2 KeyPair to enable SSH access to the instance @@ -186,27 +162,24 @@ Parameters: AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$ ConstraintDescription: must be a valid IP CIDR range of the form x.x.x.x/x. Default: 10.0.1.0/24 - Description: The address space that will be assigned to the first Splunk server - subnet. (x.x.x.x/x notation) + Description: The address space that will be assigned to the first Splunk server subnet. (x.x.x.x/x notation) Type: String PublicSubnet2CIDR: AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$ - ConstraintDescription: must be a valid IP CIDR range of the form x.x.x.x/x. + ConstraintDescription: must be a valid IP CIDR range of the form x.x.x.x/x Default: 10.0.2.0/24 - Description: The address space that will be assigned to the second Splunk server - subnet. (x.x.x.x/x notation) + Description: The address space that will be assigned to the second Splunk server subnet. (x.x.x.x/x notation) Type: String PublicSubnet3CIDR: AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$ ConstraintDescription: must be a valid IP CIDR range of the form x.x.x.x/x. Default: 10.0.3.0/24 - Description: The address space that will be assigned to the second Splunk server - subnet. (x.x.x.x/x notation) + Description: The address space that will be assigned to the second Splunk server subnet. (x.x.x.x/x notation) Type: String QSS3BucketName: - Default: aws-quickstart Description: S3 bucket name for the Quick Start assets. Type: String + Default: aws-quickstart QSS3BucketRegion: Default: 'us-west-2' Description: 'The AWS Region where the Quick Start S3 bucket (QSS3BucketName) is hosted. When using your own bucket, you must specify this value.' @@ -220,58 +193,58 @@ Parameters: - 'yes' - 'no' Default: 'no' - Description: Do you want to build a Splunk search head cluster? + Description: Do you want to build a Splunk search head cluster? Type: String SSHClientLocation: + Default: '0.0.0.0/0' AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$ - ConstraintDescription: Must be a valid IP range in x.x.x.x/x notation. Use 0.0.0.0/0 - for no restrictions. - Description: 'The IP address range that is allowed to SSH to the EC2 instances. - Note: a value of 0.0.0.0/0 will allow access from ANY ip address' + ConstraintDescription: Must be a valid IP range in x.x.x.x/x notation. Use 0.0.0.0/0 for no restrictions. + Description: The IP address range that is allowed to SSH to the EC2 instances. Note that a value of 0.0.0.0/0 will allow access from ANY ip address MaxLength: '19' MinLength: '9' Type: String SplunkAdminPassword: - AllowedPattern: (?=^.{6,255}$)((?=.*\d)(?=.*[A-Z])(?=.*[a-z])|(?=.*\d)(?=.*[^A-Za-z0-9])(?=.*[a-z])|(?=.*[^A-Za-z0-9])(?=.*[A-Z])(?=.*[a-z])|(?=.*\d)(?=.*[A-Z])(?=.*[^A-Za-z0-9]))^.* - ConstraintDescription: Must be at least 8 characters containing letters, numbers - and symbols. - Description: Admin password for Splunk. Must be at least 6 characters containing - letters, numbers and symbols + AllowedPattern: (?=.{8,})(?=.*[a-zA-Z])(?=.*[0-9]).* + ConstraintDescription: Must be at least 8 characters containing letters and numbers. + Description: Admin password for Splunk. Must be at least 8 characters containing letters and numbers. MaxLength: '32' MinLength: '6' NoEcho: 'true' Type: String SplunkIndexerCount: - ConstraintDescription: must be a valid number, 3-10 - Default: '3' - Description: How many Splunk indexers to launch. [3-10] + ConstraintDescription: must be a valid number, 4-10 + Default: '4' + Description: How many Splunk indexers to launch. [4-10] MaxValue: '10' - MinValue: '3' + MinValue: '4' Type: Number SplunkIndexerDiskSize: - ConstraintDescription: must be a valid number, 320-16000 - Default: '320' - Description: The size of the attached EBS volume to the Splunk indexers. (in - GB) + ConstraintDescription: must be a valid number, 100-16000 + Default: '334' + Description: The size of the attached EBS volume to the Splunk indexers. (in GB) MaxValue: '16000' - MinValue: '320' + MinValue: '100' Type: Number SplunkSearchHeadDiskSize: - ConstraintDescription: must be a valid number, 320-16000 - Default: '320' - Description: The size of the attached EBS volume to the Splunk search head(s). (in - GB) + ConstraintDescription: must be a valid number, 100-16000 + Default: '334' + Description: The size of the attached EBS volume to the Splunk search head(s). (in GB) MaxValue: '16000' - MinValue: '320' + MinValue: '100' Type: Number SplunkLicenseBucket: - Default: '' - Description: Name of private S3 bucket with licenses to be accessed via authenticated - requests + AllowedPattern: (?=^.{3,63}$)(?!xn--)([a-z0-9](?:[a-z0-9-]*)[a-z0-9])$ + ConstraintDescription: Required for QuickStart to function and must be a valid s3 bucket + Description: Name of private S3 bucket with licenses to be accessed via authenticated requests + MinLength: '3' + MaxLength: '63' Type: String SplunkLicensePath: - Default: '' - Description: Path to license file in S3 Bucket (without leading '/') + ConstraintDescription: Required for QuickStart to function and must point to a valid Splunk license + AllowedPattern: ([0-9]|[A-Z]|[a-z]|[\/\._-])+ + Description: Path to license file in S3 Bucket, without leading /. (for example license/splunk.license) + MinLength: '2' + MaxLength: '128' Type: String SplunkReplicationFactor: ConstraintDescription: must be a valid number, 2-4 @@ -283,30 +256,22 @@ Parameters: SplunkSearchFactor: ConstraintDescription: must be a valid number, 2-4 Default: '2' - Description: How many copies of data should be searchable in the Splunk indexer - clusters + Description: How many copies of data should be searchable in the Splunk indexer clusters MaxValue: '4' MinValue: '2' Type: Number SplunkClusterSecret: - AllowedPattern: (?=^.{6,255}$)((?=.*\d)(?=.*[A-Z])(?=.*[a-z])|(?=.*\d)(?=.*[^A-Za-z0-9])(?=.*[a-z])|(?=.*[^A-Za-z0-9])(?=.*[A-Z])(?=.*[a-z])|(?=.*\d)(?=.*[A-Z])(?=.*[^A-Za-z0-9]))^.* - ConstraintDescription: Must be at least 8 characters containing letters, numbers - and symbols. - Description: Shared cluster secret for Search Head and Indexer clusters. Must - be at least 8 characters containing letters, numbers and symbols. + AllowedPattern: (?=.{8,})(?=.*[a-zA-Z])(?=.*[0-9]).* + ConstraintDescription: Must be at least 8 characters containing letters and numbers. + Description: Shared cluster secret for Search Head and Indexer clusters. Must be at least 8 characters containing letters and numbers. MaxLength: '32' MinLength: '6' NoEcho: 'true' Type: String SplunkIndexerDiscoverySecret: - AllowedPattern: (?=^.{6,255}$)((?=.*\d)(?=.*[A-Z])(?=.*[a-z])|(?=.*\d)(?=.*[^A-Za-z0-9])(?=.*[a-z])|(?=.*[^A-Za-z0-9])(?=.*[A-Z])(?=.*[a-z])|(?=.*\d)(?=.*[A-Z])(?=.*[^A-Za-z0-9]))^.* - ConstraintDescription: Must be at least 8 characters containing letters, numbers - and symbols. - Description: >- - Security key used for communication between your forwarders and the cluster - master. This value should also be used by forwarders in order to retrieve list - of available peer nodes from cluster master. Must be at least 8 characters containing - letters, numbers and symbols. + AllowedPattern: (?=.{8,})(?=.*[a-zA-Z])(?=.*[0-9]).* + ConstraintDescription: Must be at least 8 characters containing letters and numbers. + Description: Security key used for communication between your forwarders and the cluster manager. This value should also be used by forwarders in order to retrieve list of available peer nodes from cluster manager. Must be at least 8 characters containing letters and numbers. MaxLength: '32' MinLength: '8' NoEcho: 'true' @@ -315,11 +280,13 @@ Parameters: AllowedPattern: (\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2}) ConstraintDescription: must be a valid IP CIDR range of the form x.x.x.x/x. Default: 10.0.0.0/16 - Description: The address space that will be assigned to the entire VPC where Splunk - will reside. (Recommend at least a /16) + Description: The address space that will be assigned to the entire VPC where Splunk will reside. (Recommend at least a /16) MaxLength: '19' MinLength: '9' Type: String + SmartStoreBucketName: + Description: Name of bucket that will be created for SmartStore storage + Type: String Conditions: Create3AZ: !Equals - !Ref 'NumberOfAZs' @@ -330,16 +297,16 @@ Resources: Type: AWS::CloudFormation::Stack Properties: TemplateURL: - !Sub - - 'https://${S3Bucket}.s3.${S3Region}.${AWS::URLSuffix}/${QSS3KeyPrefix}submodules/quickstart-aws-vpc/templates/aws-vpc.template' + !Sub + - https://${S3Bucket}.s3.${S3Region}.${AWS::URLSuffix}/${QSS3KeyPrefix}submodules/quickstart-aws-vpc/templates/aws-vpc.template.yaml - S3Region: !If [UsingDefaultBucket, !Ref 'AWS::Region', !Ref QSS3BucketRegion] S3Bucket: !If [UsingDefaultBucket, !Sub '${QSS3BucketName}-${AWS::Region}', !Ref QSS3BucketName] Parameters: AvailabilityZones: !Join - ',' - !Ref 'AvailabilityZones' + CreateNATGateways: 'false' CreatePrivateSubnets: 'false' - KeyPairName: !Ref 'KeyName' NumberOfAZs: !Ref 'NumberOfAZs' PublicSubnet1CIDR: !Ref 'PublicSubnet1CIDR' PublicSubnet2CIDR: !Ref 'PublicSubnet2CIDR' @@ -347,59 +314,54 @@ Resources: VPCCIDR: !Ref 'VPCCIDR' TimeoutInMinutes: 15 SplunkStack: - Type: AWS::CloudFormation::Stack - Properties: - TemplateURL: - !Sub - - 'https://${S3Bucket}.s3.${S3Region}.${AWS::URLSuffix}/${QSS3KeyPrefix}templates/splunk-enterprise.template.yaml' - - S3Region: !If [UsingDefaultBucket, !Ref 'AWS::Region', !Ref QSS3BucketRegion] - S3Bucket: !If [UsingDefaultBucket, !Sub '${QSS3BucketName}-${AWS::Region}', !Ref QSS3BucketName] - Parameters: - VPCID: !GetAtt 'VPCStack.Outputs.VPCID' - VPCCIDR: !GetAtt 'VPCStack.Outputs.VPCCIDR' - PublicSubnet1ID: !GetAtt 'VPCStack.Outputs.PublicSubnet1ID' - PublicSubnet2ID: !GetAtt 'VPCStack.Outputs.PublicSubnet2ID' - PublicSubnet3ID: !If - - Create3AZ - - !GetAtt 'VPCStack.Outputs.PublicSubnet3ID' - - !GetAtt 'VPCStack.Outputs.PublicSubnet2ID' - NumberOfAZs: !Ref 'NumberOfAZs' - IndexerInstanceType: !Ref 'IndexerInstanceType' - SearchHeadInstanceType: !Ref 'SearchHeadInstanceType' - SplunkAdminPassword: !Ref 'SplunkAdminPassword' - SplunkClusterSecret: !Ref 'SplunkClusterSecret' - SplunkIndexerDiscoverySecret: !Ref 'SplunkIndexerDiscoverySecret' - SplunkLicenseBucket: !Ref 'SplunkLicenseBucket' - SplunkLicensePath: !Ref 'SplunkLicensePath' - KeyName: !Ref 'KeyName' - SSHClientLocation: !Ref 'SSHClientLocation' - HECClientLocation: !Ref 'HECClientLocation' - WebClientLocation: !Ref 'WebClientLocation' - SplunkIndexerCount: !Ref 'SplunkIndexerCount' - SHCEnabled: !Ref 'SHCEnabled' - SplunkIndexerDiskSize: !Ref 'SplunkIndexerDiskSize' - SplunkReplicationFactor: !Ref 'SplunkReplicationFactor' - QSS3BucketName: !Ref QSS3BucketName - QSS3BucketRegion: !Ref QSS3BucketRegion - QSS3KeyPrefix: !Ref QSS3KeyPrefix - IndexerApps: !Join - - ',' - - !Ref 'IndexerApps' - SearchHeadApps: !Join - - ',' - - !Ref 'SearchHeadApps' - TimeoutInMinutes: 60 + Type: AWS::CloudFormation::Stack + Properties: + TemplateURL: + !Sub + - https://${S3Bucket}.s3.${S3Region}.${AWS::URLSuffix}/${QSS3KeyPrefix}templates/splunk-enterprise.template.yaml + - S3Region: !If [UsingDefaultBucket, !Ref 'AWS::Region', !Ref QSS3BucketRegion] + S3Bucket: !If [UsingDefaultBucket, !Sub '${QSS3BucketName}-${AWS::Region}', !Ref QSS3BucketName] + Parameters: + VPCID: !GetAtt 'VPCStack.Outputs.VPCID' + VPCCIDR: !GetAtt 'VPCStack.Outputs.VPCCIDR' + PublicSubnet1ID: !GetAtt 'VPCStack.Outputs.PublicSubnet1ID' + PublicSubnet2ID: !GetAtt 'VPCStack.Outputs.PublicSubnet2ID' + PublicSubnet3ID: !If + - Create3AZ + - !GetAtt 'VPCStack.Outputs.PublicSubnet3ID' + - !GetAtt 'VPCStack.Outputs.PublicSubnet2ID' + NumberOfAZs: !Ref 'NumberOfAZs' + IndexerInstanceType: !Ref 'IndexerInstanceType' + SearchHeadInstanceType: !Ref 'SearchHeadInstanceType' + SplunkAdminPassword: !Ref 'SplunkAdminPassword' + SplunkClusterSecret: !Ref 'SplunkClusterSecret' + SplunkIndexerDiscoverySecret: !Ref 'SplunkIndexerDiscoverySecret' + SplunkLicenseBucket: !Ref 'SplunkLicenseBucket' + SplunkLicensePath: !Ref 'SplunkLicensePath' + KeyName: !Ref 'KeyName' + SSHClientLocation: !Ref 'SSHClientLocation' + HECClientLocation: !Ref 'HECClientLocation' + WebClientLocation: !Ref 'WebClientLocation' + SplunkIndexerCount: !Ref 'SplunkIndexerCount' + SHCEnabled: !Ref 'SHCEnabled' + SplunkIndexerDiskSize: !Ref 'SplunkIndexerDiskSize' + SplunkSearchHeadDiskSize: !Ref 'SplunkSearchHeadDiskSize' + SmartStoreBucketName: !Ref 'SmartStoreBucketName' + SplunkReplicationFactor: !Ref 'SplunkReplicationFactor' + SplunkSearchFactor: !Ref 'SplunkSearchFactor' + QSS3KeyPrefix: !Ref 'QSS3KeyPrefix' + QSS3BucketName: !Ref 'QSS3BucketName' + TimeoutInMinutes: 45 Outputs: SearchHeadURL: Description: Splunk Enterprise - Search Head URL Value: !GetAtt 'SplunkStack.Outputs.SearchHeadURL' - ClusterMasterURL: - Description: Splunk Enterprise - Cluster Master URL - Value: !GetAtt 'SplunkStack.Outputs.ClusterMasterURL' - ClusterMasterManagementURL: - Description: Splunk Enterprise - Cluster Master Management URL (required for Indexer - Discovery) - Value: !GetAtt 'SplunkStack.Outputs.ClusterMasterManagementURL' + ClusterManagerURL: + Description: Splunk Enterprise - Cluster Manager URL + Value: !GetAtt 'SplunkStack.Outputs.ClusterManagerURL' + ClusterManagerManagementURL: + Description: Splunk Enterprise - Cluster Manager Management URL (required for Indexer Discovery) + Value: !GetAtt 'SplunkStack.Outputs.ClusterManagerManagementURL' DeployerURL: Description: Splunk Enterprise - Search Head Cluster Deployer URL Value: !GetAtt 'SplunkStack.Outputs.DeployerURL' diff --git a/templates/splunk-enterprise.template.yaml b/templates/splunk-enterprise.template.yaml index 1a1c5de..ffcc196 100644 --- a/templates/splunk-enterprise.template.yaml +++ b/templates/splunk-enterprise.template.yaml @@ -1,182 +1,61 @@ AWSTemplateFormatVersion: '2010-09-09' -Description: Splunk deployment with indexer, search head clustering and cluster master. - (qs-1qup6ramp) -Metadata: - QuickStartDocumentation: - EntrypointName: "Parameters for an existing VPC" - AWSAMIRegionMap: - Filters: - SPLUNKENTHVM: - name: splunk_marketplace_AMI_* - owner-alias: aws-marketplace - product-code.type: marketplace - AWS::CloudFormation::Interface: - ParameterGroups: - - Label: - default: AWS Instance and Network Settings - Parameters: - - IndexerInstanceType - - SearchHeadInstanceType - - KeyName - - WebClientLocation - - HECClientLocation - - SSHClientLocation - - VPCID - - VPCCIDR - - PublicSubnet1ID - - PublicSubnet2ID - - PublicSubnet3ID - - NumberOfAZs - - Label: - default: Splunk Settings - Parameters: - - SplunkAdminPassword - - SplunkClusterSecret - - SplunkIndexerDiscoverySecret - - SplunkLicenseBucket - - SplunkLicensePath - - SplunkIndexerCount - - SplunkIndexerDiskSize - - SplunkSearchHeadDiskSize - - SplunkReplicationFactor - - SplunkSearchFactor - - SHCEnabled - - IndexerApps - - SearchHeadApps - ParameterLabels: - QSS3BucketName: - default: QuickStart S3 Bucket Name - QSS3BucketRegion: - default: Quick Start S3 bucket region - QSS3KeyPrefix: - default: QuickStart S3 Key Prefix - WebClientLocation: - default: Permitted CIDR for Splunk web interface - HECClientLocation: - default: Permitted CIDR for Splunk HTTP event collector input - IndexerInstanceType: - default: EC2 instance type for Splunk indexer - SearchHeadInstanceType: - default: EC2 instance type for Splunk search head - KeyName: - default: Key Name - PublicSubnet1ID: - default: Public Subnet 1 ID - PublicSubnet2ID: - default: Public Subnet 2 ID - PublicSubnet3ID: - default: Public Subnet 3 ID - NumberOfAZs: - default: Number of Availability Zones - SHCEnabled: - default: Enable Search Head Cluster? - SSHClientLocation: - default: Permitted CIDR for ssh - SplunkAdminPassword: - default: Splunk Admin Password - SplunkIndexerCount: - default: No. of Splunk Indexers - SplunkIndexerDiskSize: - default: Indexer Disk Size - SplunkSearchHeadDiskSize: - default: Search Head(s) Disk Size - SplunkLicenseBucket: - default: Splunk License Bucket - SplunkLicensePath: - default: Splunk License S3 Bucket Path - SplunkReplicationFactor: - default: Index Cluster Replication Factor - SplunkSearchFactor: - default: Index Cluster Search Factor - SplunkClusterSecret: - default: Shared Security Key for Cluster Nodes - SplunkIndexerDiscoverySecret: - default: Shared Security Key for Forwarders using Indexer Discovery - IndexerApps: - default: Apps/Add-ons to pre-Install on Splunk Indexers - SearchHeadApps: - default: Apps/Add-ons to pre-Install on Splunk Search Heads - VPCCIDR: - default: VPC CIDR - VPCID: - default: VPC ID +Description: Splunk deployment with indexer clustering, search head with optional search head clustering, and cluster manager/license server. (qs-1qup6ramp) Parameters: WebClientLocation: AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$ - ConstraintDescription: Must be a valid IP range in x.x.x.x/x notation. Use 0.0.0.0/0 - for no restrictions. - Description: 'The IP address range that is allowed to connect to the Splunk web - interface. Note: a value of 0.0.0.0/0 will allow access from ANY ip address' + ConstraintDescription: Must be a valid IP range in x.x.x.x/x notation. Use 0.0.0.0/0 for no restrictions. + Description: The IP address range that is allowed to connect to the Splunk web interface. Note that a value of 0.0.0.0/0 will allow access from ANY ip address MaxLength: '19' MinLength: '9' Type: String HECClientLocation: AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$ - ConstraintDescription: Must be a valid IP range in x.x.x.x/x notation. Use 0.0.0.0/0 - for no restrictions. - Description: 'The IP address range that is allowed to send data to Splunk HTTP - Event Collector. Note: a value of 0.0.0.0/0 will allow access from ANY ip address' + ConstraintDescription: Must be a valid IP range in x.x.x.x/x notation. Use 0.0.0.0/0 for no restrictions. + Description: The IP address range that is allowed to send data to Splunk HTTP Event Collector. Note that a value of 0.0.0.0/0 will allow access from ANY ip address MaxLength: '19' MinLength: '9' Type: String IndexerInstanceType: AllowedValues: - - c4.2xlarge - - c4.4xlarge - - c4.8xlarge - - m4.2xlarge - - m4.4xlarge - - m4.10xlarge - - c5.2xlarge + - m5.4xlarge + - m5.8xlarge - c5.4xlarge - c5.9xlarge - c5.18xlarge - - i3.2xlarge - i3.4xlarge - i3.8xlarge + - i3en.3xlarge + - i3en.6xlarge + - i3en.12xlarge Description: EC2 instance type for Splunk Indexers - ConstraintDescription: must be a valid EC2 instance type. - Default: c5.4xlarge + ConstraintDescription: Must be a valid EC2 instance type. + Default: i3.4xlarge Type: String SearchHeadInstanceType: AllowedValues: - - c4.2xlarge - - c4.4xlarge - - c4.8xlarge - - r4.4xlarge - - r4.8xlarge - - r4.16xlarge - - c5.2xlarge + - r5.4xlarge + - r5.8xlarge + - r5.16xlarge - c5.4xlarge - c5.9xlarge - m5.2xlarge - m5.4xlarge + - m5.8xlarge - m5.12xlarge Description: EC2 instance type for Splunk Search Heads - ConstraintDescription: must be a valid EC2 instance type. + ConstraintDescription: Must be a valid EC2 instance type. Default: c5.4xlarge Type: String - IndexerApps: - Description: Comma separated list of URLs of Splunk App (or Add-on) tarballs (.spl) - to pre-install on indexer(s) - Default: '' - Type: CommaDelimitedList - SearchHeadApps: - Description: Comma separated list of URLs of Splunk App (or Add-on) tarballs (.spl) - to pre-install on search head(s) - Default: '' - Type: CommaDelimitedList KeyName: ConstraintDescription: Must be the name of an existing EC2 KeyPair. - Description: Name of an existing EC2 KeyPair to enable SSH access to the instance + Description: Name of an existing EC2 KeyPair to enable SSH access to the instance. Type: AWS::EC2::KeyPair::KeyName NumberOfAZs: AllowedValues: - '2' - '3' Default: '2' - Description: Number of Availability Zones to use in the VPC. This must match the - number public subnet IDs entered as parameters + Description: Number of Availability Zones to use in the VPC. This must match the number public subnet IDs entered as parameters. Type: String PublicSubnet1ID: Description: ID of Splunk public subnet 1 in Availability Zone 1 (e.g., subnet-xxxxxxxx) @@ -187,15 +66,10 @@ Parameters: PublicSubnet3ID: Description: ID of Splunk public subnet 3 in Availability Zone 3 (e.g., subnet-xxxxxxxx) Type: AWS::EC2::Subnet::Id - Default: '' QSS3BucketName: - Default: splk-quickstart-testing + Default: aws-quickstart Description: S3 bucket name for the Quick Start assets. Type: String - QSS3BucketRegion: - Default: 'us-east-1' - Description: 'The AWS Region where the Quick Start S3 bucket (QSS3BucketName) is hosted. When using your own bucket, you must specify this value.' - Type: String QSS3KeyPrefix: Default: quickstart-splunk-enterprise/ Description: S3 key prefix for the Quick Start assets. @@ -209,101 +83,186 @@ Parameters: Type: String SSHClientLocation: AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$ - ConstraintDescription: Must be a valid IP range in x.x.x.x/x notation. Use 0.0.0.0/0 - for no restrictions. - Description: 'The IP address range that is allowed to SSH to the EC2 instances. - Note: a value of 0.0.0.0/0 will allow access from ANY ip address' + ConstraintDescription: Must be a valid IP range in x.x.x.x/x notation. Use 0.0.0.0/0 for no restrictions. + Description: The IP address range that is allowed to SSH to the EC2 instances. Note that a value of 0.0.0.0/0 will allow access from ANY ip address MaxLength: '19' MinLength: '9' Type: String SplunkAdminPassword: - AllowedPattern: (?=^.{6,255}$)((?=.*\d)(?=.*[A-Z])(?=.*[a-z])|(?=.*\d)(?=.*[^A-Za-z0-9])(?=.*[a-z])|(?=.*[^A-Za-z0-9])(?=.*[A-Z])(?=.*[a-z])|(?=.*\d)(?=.*[A-Z])(?=.*[^A-Za-z0-9]))^.* - ConstraintDescription: Must be at least 8 characters containing letters, numbers - and symbols. - Description: Admin password for Splunk. Must be at least 6 characters containing - letters, numbers and symbols. + AllowedPattern: (?=.{8,})(?=.*[a-zA-Z])(?=.*[0-9]).* + ConstraintDescription: Must be at least 8 characters containing letters, numbers and symbols. + Description: Admin password for Splunk. Must be at least 8 characters containing letters, numbers and symbols. MaxLength: '32' - MinLength: '6' + MinLength: '8' NoEcho: 'true' Type: String SplunkIndexerCount: - ConstraintDescription: must be a valid number, 3-10 - Default: '3' - Description: How many Splunk indexers to launch. [3-10] + ConstraintDescription: Must be a valid number, 4-10 + Default: '4' + Description: How many Splunk indexers to launch. [4-10] MaxValue: '10' - MinValue: '3' + MinValue: '4' Type: Number SplunkIndexerDiskSize: - ConstraintDescription: must be a valid number, 320-16000 - Default: '320' - Description: The size of the attached EBS volume to the Splunk indexers. (in - GB) + ConstraintDescription: Must be a valid number, 100-16000 + Default: '300' + Description: The size of the attached EBS volume to the Splunk indexers. (in GB) MaxValue: '16000' - MinValue: '320' + MinValue: '100' Type: Number SplunkSearchHeadDiskSize: - ConstraintDescription: must be a valid number, 320-16000 - Default: '320' - Description: The size of the attached EBS volume to the Splunk search head(s). (in - GB) + ConstraintDescription: Must be a valid number, 100-16000 + Default: '300' + Description: The size of the attached EBS volume to the Splunk search head(s). (in GB) MaxValue: '16000' - MinValue: '320' + MinValue: '100' Type: Number - SplunkLicenseBucket: + SmartStoreBucketName: Default: '' - Description: Name of private S3 bucket with licenses to be accessed via authenticated - requests + Description: Name of S3 bucket to be created for SmartStore storage + Type: String + SplunkLicenseBucket: + ConstraintDescription: Required for QuickStart to function and must be a valid s3 bucket + AllowedPattern: (?=^.{3,63}$)(?!xn--)([a-z0-9](?:[a-z0-9-]*)[a-z0-9])$ + Description: Name of private S3 bucket with licenses to be accessed via authenticated requests + MinLength: '3' + MaxLength: '63' Type: String SplunkLicensePath: - Default: '' - Description: Path to license file in S3 Bucket (without leading '/') + ConstraintDescription: Required for QuickStart to function and must point to a valid Splunk license + AllowedPattern: ([0-9]|[A-Z]|[a-z]|[\/\._-])+ + Description: Path to license file in S3 Bucket (without leading /) + MinLength: '1' + MaxLength: '128' Type: String SplunkReplicationFactor: - ConstraintDescription: must be a valid number, 2-4 + ConstraintDescription: Must be a valid number, 2-6 Default: '2' Description: How many copies of data should be stored in the Splunk Indexer Cluster - MaxValue: '4' + MaxValue: '6' MinValue: '2' Type: Number SplunkSearchFactor: - ConstraintDescription: must be a valid number, 2-4 + ConstraintDescription: Must be a valid number, 2-6 Default: '2' - Description: How many copies of data should be searchable in the Splunk indexer - clusters - MaxValue: '4' + Description: How many copies of data should be searchable in the Splunk indexer clusters + MaxValue: '6' MinValue: '2' Type: Number SplunkClusterSecret: - AllowedPattern: (?=^.{6,255}$)((?=.*\d)(?=.*[A-Z])(?=.*[a-z])|(?=.*\d)(?=.*[^A-Za-z0-9])(?=.*[a-z])|(?=.*[^A-Za-z0-9])(?=.*[A-Z])(?=.*[a-z])|(?=.*\d)(?=.*[A-Z])(?=.*[^A-Za-z0-9]))^.* - ConstraintDescription: Must be at least 8 characters containing letters, numbers - and symbols. - Description: Shared cluster secret for Search Head and Indexer cluster nodes. - Must be at least 8 characters containing letters, numbers and symbols. + AllowedPattern: (?=.{8,})(?=.*[a-zA-Z])(?=.*[0-9]).* + ConstraintDescription: Must be at least 8 characters containing letters, numbers and symbols. + Description: Shared cluster secret for Search Head and Indexer cluster nodes. Must be at least 8 characters containing letters, numbers and symbols. MaxLength: '32' MinLength: '8' NoEcho: 'true' Type: String SplunkIndexerDiscoverySecret: - AllowedPattern: (?=^.{6,255}$)((?=.*\d)(?=.*[A-Z])(?=.*[a-z])|(?=.*\d)(?=.*[^A-Za-z0-9])(?=.*[a-z])|(?=.*[^A-Za-z0-9])(?=.*[A-Z])(?=.*[a-z])|(?=.*\d)(?=.*[A-Z])(?=.*[^A-Za-z0-9]))^.* - ConstraintDescription: Must be at least 8 characters containing letters, numbers - and symbols. - Description: >- - Security key used for communication between your forwarders and the cluster - master. This value should also be used by forwarders in order to retrieve list - of available peer nodes from cluster master. Must be at least 8 characters containing - letters, numbers and symbols. + AllowedPattern: (?=.{8,})(?=.*[a-zA-Z])(?=.*[0-9]).* + ConstraintDescription: Must be at least 8 characters containing letters, numbers and symbols. + Description: Security key used for communication between your forwarders and the cluster manager. This value should also be used by forwarders in order to retrieve list of available peer nodes from cluster manager. Must be at least 8 characters containing letters, numbers and symbols. MaxLength: '32' MinLength: '8' NoEcho: 'true' Type: String VPCCIDR: AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$ - ConstraintDescription: must be a valid IP CIDR range of the form x.x.x.x/x. + ConstraintDescription: Must be a valid IP CIDR range of the form x.x.x.x/x. Description: VPC CIDR Block (x.x.x.x/x notation) Type: String VPCID: Description: VPC ID Type: AWS::EC2::VPC::Id +Metadata: + AWSAMIRegionMap: + Filters: + SPLUNKENTHVM: + name: splunk_AMI* + owner-alias: aws-marketplace + product-code.type: marketplace + QuickStartDocumentation: + EntrypointName: "Splunk QuickStart (Existing VPC)" + AWS::CloudFormation::Interface: + ParameterGroups: + - Label: + default: AWS Instance and Network Settings + Parameters: + - IndexerInstanceType + - SearchHeadInstanceType + - KeyName + - WebClientLocation + - HECClientLocation + - SSHClientLocation + - VPCID + - VPCCIDR + - PublicSubnet1ID + - PublicSubnet2ID + - PublicSubnet3ID + - NumberOfAZs + - Label: + default: Splunk Settings + Parameters: + - SplunkAdminPassword + - SplunkClusterSecret + - SplunkIndexerDiscoverySecret + - SplunkLicenseBucket + - SplunkLicensePath + - SplunkIndexerCount + - SplunkIndexerDiskSize + - SplunkSearchHeadDiskSize + - SplunkReplicationFactor + - SplunkSearchFactor + - SmartStoreBucketName + - SHCEnabled + ParameterLabels: + WebClientLocation: + default: Permitted CIDR for Splunk web interface + HECClientLocation: + default: Permitted CIDR for Splunk HTTP event collector input + IndexerInstanceType: + default: EC2 instance type for Splunk indexer + SearchHeadInstanceType: + default: EC2 instance type for Splunk search head + KeyName: + default: Key Name + PublicSubnet1ID: + default: Public Subnet 1 ID + PublicSubnet2ID: + default: Public Subnet 2 ID + PublicSubnet3ID: + default: Public Subnet 3 ID + NumberOfAZs: + default: Number of Availability Zones + SHCEnabled: + default: Enable Search Head Cluster? + SSHClientLocation: + default: Permitted CIDR for ssh + SplunkAdminPassword: + default: Splunk Admin Password + SplunkIndexerCount: + default: No. of Splunk Indexers + SmartStoreBucketName: + default: Name of bucket to be created for Smartstore storage + SplunkIndexerDiskSize: + default: Indexer Disk Size + SplunkSearchHeadDiskSize: + default: Search Head(s) Disk Size + SplunkLicenseBucket: + default: Splunk License Bucket + SplunkLicensePath: + default: Splunk License S3 Bucket Path + SplunkReplicationFactor: + default: Index Cluster Replication Factor + SplunkSearchFactor: + default: Index Cluster Search Factor + SplunkClusterSecret: + default: Shared Security Key for Cluster Nodes + SplunkIndexerDiscoverySecret: + default: Shared Security Key for Forwarders using Indexer Discovery + VPCCIDR: + default: VPC CIDR + VPCID: + default: VPC ID Conditions: Create3AZ: !Equals - !Ref 'NumberOfAZs' @@ -314,18 +273,6 @@ Conditions: CreateSHC: !Equals - !Ref 'SHCEnabled' - 'yes' - InstallIndexerApps: !Not - - !Equals - - !Join - - '' - - !Ref 'IndexerApps' - - '' - InstallSearchHeadApps: !Not - - !Equals - - !Join - - '' - - !Ref 'SearchHeadApps' - - '' ConfigureLicense: !And - !Not - !Equals @@ -335,55 +282,113 @@ Conditions: - !Equals - '' - !Ref 'SplunkLicensePath' - UsingDefaultBucket: !Equals [!Ref QSS3BucketName, 'aws-quickstart'] Mappings: AWSAMIRegionMap: AMI: - SPLUNKENTHVM: splunk_marketplace_AMI_2018-10-16_22_07_36-7b65de6c-5006-4ca2-bd75-fdba95ae5d9d-ami-0d494b5a999e1c49f.4 + SPLUNKENTHVM: splunk_AMI_8.2.6_2022-04-20_17-27-55-7b65de6c-5006-4ca2-bd75-fdba95ae5d9d + us-west-1: + SPLUNKENTHVM: ami-083c40352f95a2a8d + us-west-2: + SPLUNKENTHVM: ami-063db914927fd068a + us-east-1: + SPLUNKENTHVM: ami-02be4afccabbc98a7 + us-east-2: + SPLUNKENTHVM: ami-0e3657d8025d1f571 + ap-south-1: + SPLUNKENTHVM: ami-04f8e05450860a0e4 ap-northeast-1: - SPLUNKENTHVM: ami-0db36f11d65f551fb + SPLUNKENTHVM: ami-0316ee1a35129c1ed ap-northeast-2: - SPLUNKENTHVM: ami-09c7965888207979b - ap-south-1: - SPLUNKENTHVM: ami-07c20db6edfd45f98 + SPLUNKENTHVM: ami-0441f1c60fa588b30 ap-southeast-1: - SPLUNKENTHVM: ami-0e7b7ca1bdcdd93a6 + SPLUNKENTHVM: ami-0066d2b4e4446b6b4 ap-southeast-2: - SPLUNKENTHVM: ami-0c8a4d5bdf83f0df8 + SPLUNKENTHVM: ami-0b93708f93e6fcefa ca-central-1: - SPLUNKENTHVM: ami-02f085f4514fa7145 + SPLUNKENTHVM: ami-0287afc9fe6de24dc eu-central-1: - SPLUNKENTHVM: ami-09ce965c3b1a9a1cb + SPLUNKENTHVM: ami-0258ed74a65c3aa80 eu-west-1: - SPLUNKENTHVM: ami-0fafe9e81915f154e + SPLUNKENTHVM: ami-0ef490267313f2958 eu-west-2: - SPLUNKENTHVM: ami-060d9e50d310e0ebb + SPLUNKENTHVM: ami-0eed7148a3c5db6a0 + eu-west-3: + SPLUNKENTHVM: ami-0f8ed68391019e14a + eu-north-1: + SPLUNKENTHVM: ami-0c13450402b8e9f7a sa-east-1: - SPLUNKENTHVM: ami-0dacd4005280936e5 - us-east-1: - SPLUNKENTHVM: ami-0484972f36720ea7f - us-east-2: - SPLUNKENTHVM: ami-04b6874c649721f0a - us-west-1: - SPLUNKENTHVM: ami-0377011a3f771e353 - us-west-2: - SPLUNKENTHVM: ami-0c3e33232b6c07537 + SPLUNKENTHVM: ami-02f1a54ae0463d7ac SplunkConfig: dedicated-instance-type: - clusterMaster: c5.xlarge + clusterManager: c5.xlarge shclusterDeployer: c5.xlarge shcluster-replication-factor: num: '3' labels: cluster: IndexerCluster shcluster: SearchHeadCluster + Resources: + SplunkSmartstoreBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Ref 'SmartStoreBucketName' + BucketEncryption: + ServerSideEncryptionConfiguration: + - ServerSideEncryptionByDefault: + SSEAlgorithm: AES256 + DeletionPolicy: Delete + SmartStoreS3BucketRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Statement: + - Effect: Allow + Principal: + Service: + - ec2.amazonaws.com + Action: + - sts:AssumeRole + Path: / + SmartStoreS3AccessInstanceProfile: + Type: AWS::IAM::InstanceProfile + Properties: + Path: / + Roles: + - !Ref 'SmartStoreS3BucketRole' + SmartStoreS3BucketPolicy: + Type: AWS::IAM::Policy + Properties: + PolicyName: SmartStoreS3BucketPolicy + PolicyDocument: + Statement: + - Action: + - s3:ListBucket + Effect: Allow + Resource: + - !Join + - '' + - - 'arn:aws:s3:::' + - !Ref 'SmartStoreBucketName' + - Action: + - s3:PutObject + - s3:GetObject + - s3:DeleteObject + - s3:PutObjectAcl + Effect: Allow + Resource: + - !Join + - '' + - - 'arn:aws:s3:::' + - !Ref 'SmartStoreBucketName' + - '/*' + Roles: + - !Ref 'SmartStoreS3BucketRole' SplunkSearchHeadSecurityGroup: Type: AWS::EC2::SecurityGroup Properties: VpcId: !Ref 'VPCID' - GroupDescription: Enable port 8000 for Splunk web interface, port 8090 for SHC - replication, and port 8191 for KV store replication + GroupDescription: Enable port 8000 for Splunk web interface, port 8090 for SHC replication, and port 8191 for KV store replication SecurityGroupIngress: - IpProtocol: tcp FromPort: 8000 @@ -406,8 +411,7 @@ Resources: Type: AWS::EC2::SecurityGroup Properties: VpcId: !Ref 'VPCID' - GroupDescription: Enable port 9997 for splunktcp input, port 8088 for HEC input, - port 514 for tcp/udp input, and port 9887 for data replication + GroupDescription: Enable port 9997 for splunktcp input, port 8088 for HEC input, port 514 for tcp/udp input, and port 9887 for data replication SecurityGroupIngress: - IpProtocol: tcp FromPort: 9997 @@ -438,8 +442,7 @@ Resources: Type: AWS::EC2::SecurityGroup Properties: VpcId: !Ref 'VPCID' - GroupDescription: Enable administrative ports like restricted SSH and management - port + GroupDescription: Enable administrative ports like restricted SSH and management port SecurityGroupIngress: - IpProtocol: tcp FromPort: 22 @@ -474,7 +477,30 @@ Resources: Condition: CreateSingleSearchHead CreationPolicy: ResourceSignal: - Timeout: PT60M + Timeout: PT15M + Metadata: + AWS::CloudFormation::Init: + config: + files: + /tmp/user_data.sh: + source: !Join + - '' + - - https:// + - !Ref 'QSS3BucketName' + - .s3.amazonaws.com/ + - !Ref 'QSS3KeyPrefix' + - scripts/user_data.sh + mode: '000755' + owner: root + group: root + authentication: S3AccessCreds + AWS::CloudFormation::Authentication: + S3AccessCreds: + type: S3 + accessKeyId: !Ref 'CfnKeys' + secretKey: !GetAtt 'CfnKeys.SecretAccessKey' + buckets: + - !Ref 'QSS3BucketName' Properties: ImageId: !FindInMap - AWSAMIRegionMap @@ -500,123 +526,44 @@ Resources: BlockDeviceMappings: - DeviceName: /dev/xvda Ebs: - VolumeType: gp2 + VolumeType: gp3 VolumeSize: !Ref 'SplunkSearchHeadDiskSize' - UserData: !Base64 - Fn::Join: - - '' - - - "#!/bin/bash -v\n" - - "# First make cloud-init output log readable by root only to protect\ - \ sensitive parameter values\n" - - "chmod 600 /var/log/cloud-init-output.log\n" - - "yum update -y aws-cfn-bootstrap\n" - - "export LOCALIP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)\n" - - "export INSTANCEID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id)\n" - - "export SPLUNK_USER=splunk\n" - - "export SPLUNK_BIN=/opt/splunk/bin/splunk\n" - - "export SPLUNK_HOME=/opt/splunk\n" - - "printf '%s\t%s\n' \"$LOCALIP\" 'splunksearch' >> /etc/hosts\n" - - "hostname splunksearch\n" - - "mv $SPLUNK_HOME/etc/passwd $SPLUNK_HOME/etc/passwd.bak\n" - - "cat >>$SPLUNK_HOME/etc/system/local/user-seed.conf <>$SPLUNK_HOME/etc/apps/base-autogenerated/local/web.conf <>$SPLUNK_HOME/etc/apps/base-autogenerated/local/outputs.conf </dev/null)\n" - - "export SPLUNK_USER=splunk\n" - - "export SPLUNK_BIN=/opt/splunk/bin/splunk\n" - - "export SPLUNK_HOME=/opt/splunk\n" - - "# remove stale splunkd.log that ships with AMI.\n" - - "rm -f $SPLUNK_HOME/var/log/splunk/splunkd.log\n" - - "printf '%s\t%s\n' \"$LOCALIP\" 'splunklicense' >> /etc/hosts\n" - - "hostname splunklicense\n" - - "mv $SPLUNK_HOME/etc/passwd $SPLUNK_HOME/etc/passwd.bak\n" - - "cat >>$SPLUNK_HOME/etc/system/local/user-seed.conf <>$SPLUNK_HOME/etc/apps/base-autogenerated/local/web.conf <>$SPLUNK_HOME/etc/apps/base-autogenerated/local/outputs.conf <>$SPLUNK_HOME/etc/system/local/server.conf < /tmp/token\n" - - "TOKEN=`sed -n 's/\\ttoken=//p' /tmp/token` && rm /tmp/token\n" - - "echo $TOKEN\n" - - "mkdir -p $SPLUNK_HOME/etc/master-apps/peer-base-autogenerated/local\n" - - "mv $SPLUNK_HOME/etc/apps/splunk_httpinput/local/inputs.conf $SPLUNK_HOME/etc/master-apps/peer-base-autogenerated/local\n" - - "# Peer config 2: Enable splunktcp input\n" - - "cat >>$SPLUNK_HOME/etc/master-apps/peer-base-autogenerated/local/inputs.conf\ - \ <> /etc/hosts\n" - - "hostname splunk-shc-deployer\n" - - "mv $SPLUNK_HOME/etc/passwd $SPLUNK_HOME/etc/passwd.bak\n" - - "cat >>$SPLUNK_HOME/etc/system/local/user-seed.conf <>$SPLUNK_HOME/etc/apps/base-autogenerated/local/web.conf <>$SPLUNK_HOME/etc/apps/base-autogenerated/local/server.conf <>$SPLUNK_HOME/etc/apps/base-autogenerated/local/outputs.conf <>$SPLUNK_HOME/etc/shcluster/apps/member-base-autogenerated/local/outputs.conf\ - \ <> /etc/hosts\n" - - "hostname splunksearch\n" - - "mv $SPLUNK_HOME/etc/passwd $SPLUNK_HOME/etc/passwd.bak\n" - - "cat >>$SPLUNK_HOME/etc/system/local/user-seed.conf <$SPLUNK_HOME/etc/system/local/web.conf <>$SPLUNK_HOME/etc/system/local/server.conf <> /etc/hosts\n" - - "hostname splunksearch\n" - - "mv $SPLUNK_HOME/etc/passwd $SPLUNK_HOME/etc/passwd.bak\n" - - "cat >>$SPLUNK_HOME/etc/system/local/user-seed.conf <$SPLUNK_HOME/etc/system/local/web.conf <>$SPLUNK_HOME/etc/system/local/server.conf <> /etc/hosts\n" - - "hostname splunksearch\n" - - "mv $SPLUNK_HOME/etc/passwd $SPLUNK_HOME/etc/passwd.bak\n" - - "cat >>$SPLUNK_HOME/etc/system/local/user-seed.conf <$SPLUNK_HOME/etc/system/local/web.conf <>$SPLUNK_HOME/etc/system/local/server.conf <>$SPLUNK_HOME/etc/system/local/user-seed.conf <>$SPLUNK_HOME/etc/apps/base-autogenerated/local/web.conf <>$SPLUNK_HOME/etc/apps/base-autogenerated/local/server.conf <