diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fb351f4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +config/newrelic_plugin.yml +.bundle +vendor/ +Gemfile.lock diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..34219f6 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,65 @@ +## New Relic Amazon CloudWatch Plugin ## + +### v3.3.5 - October 19, 2016 ### + +**Changes** + +* Support new namespace: AWS/ApplicationELB + +### v3.3.4 - July 21, 2016 ### + +**Changes** + +* Support new region: ap-south-1 + +### v3.3.3 - February 21, 2016 ### + +**Changes** + +* Update AWS SDK and support new regions: ap-northeast-2, eu-central-1 + +### v3.3.2 - September 25, 2014 ### + +**Bug Fixes** + +* Fixed a Ruby 1.8.7 bug that was introduced in v3.3.1 + +### v3.3.1 - September 23, 2014 ### + +**Features** + +* Added support for pulling AWS credentials from IAM Roles and Instance Profiles +* Added support for filtering RDS instances by instance identifiers + +**Bug Fixes** + +* Fixed cloudwatch_delay bug for EC2, EBS, SNS, and SQS where end_time was being set incorrectly + +### v3.3.0 - January 22, 2014 ### + +**Features** + +* Added AWS tag filtering for EC2 and EBS instances +* Added new metrics to ELB: `BackendConnectionErrors`, `SurgeQueueLength`, and `SpilloverCount` +* Added new `ECR` plugin support for ElastiCache Redis + +**Bug Fixes** + +* Fixed detailed one minute monitoring for EC2 instances + +**Changes** + +* Overview dashboards have been deprecated + +### v3.2.1 - December 16, 2013 ### + +**Features** + +* Added `cloudwatch_delay` attribute to configure each service. See README for details +* Added AWS SDK debug logging at verbose log level 2 or higher + +### v3.2.0 - November 6, 2013 ### + +**Features** + +* Upgraded AWS SDK to 1.24.0 for general improvements diff --git a/Gemfile b/Gemfile index ab04e6a..24daaff 100644 --- a/Gemfile +++ b/Gemfile @@ -1,3 +1,5 @@ -source "http://rubygems.org" -gem "newrelic_plugin", :git => "git@github.com:newrelic-platform/newrelic_plugin.git" - +source "https://rubygems.org" +gem 'newrelic_plugin', '~> 1.3.0' +gem 'nokogiri', '<= 1.5.9' +gem 'aws-sdk', '~> 1.66.0' +gem 'daemons' diff --git a/Gemfile.lock b/Gemfile.lock deleted file mode 100644 index 3773aa1..0000000 --- a/Gemfile.lock +++ /dev/null @@ -1,19 +0,0 @@ -GIT - remote: git@github.com:newrelic-platform/newrelic_plugin.git - revision: 345deb7a8508680eb54bab0119a9e8c52df484e1 - specs: - newrelic_plugin (0.2.9) - faraday (>= 0.8.1) - -GEM - remote: http://rubygems.org/ - specs: - faraday (0.8.4) - multipart-post (~> 1.1) - multipart-post (1.1.5) - -PLATFORMS - ruby - -DEPENDENCIES - newrelic_plugin! diff --git a/README.md b/README.md index dd260c1..76ba3ab 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,195 @@ -## Example Agent Install - -1. Download the latest tagged version from `https://github.com/newrelic-platform/newrelic_example_plugin/tags` -2. Extract to the location you want to run the example agent from -3. Copy `config/template_newrelic_plugin.yml` to `config/newrelic_plugin.yml` -4. Edit `config/newrelic_plugin.yml` and replace "YOUR_LICENSE_KEY_HERE" with your New Relic license key -5. Create a plugin in New Relic -6. Edit `newrelic_example_agent` and replace "PUT YOUR GUID HERE" with the GUID that was generated when you created the plugin -7. run `./newrelic_example_agent` +# New Relic Amazon CloudWatch Plugin + +This tool provides the metric collection agents for the following New Relic plugins: + +- EC2 +- EBS +- ELB +- RDS +- SQS +- SNS +- ElastiCache + - Memcached + - Redis + +## Dependencies +- A single t1.micro EC2 instance (in any region) +- Ruby (>= 1.9.2) +- Rubygems (>= 1.3.7) +- Bundler `gem install bundler` +- Git + +## Install + +### Manual Installation +1. Download the latest tagged version from [https://github.com/newrelic-platform/newrelic_aws_cloudwatch_extension/tags](https://github.com/newrelic-platform/newrelic_aws_cloudwatch_extension/tags) +2. Extract to the location you want to run the plugin from +3. Rename `config/template_newrelic_plugin.yml` to `config/newrelic_plugin.yml` +4. Edit `config/newrelic_plugin.yml` +5. Run `bundle install` +6. Run `bundle exec ./bin/newrelic_aws` (or on Windows `bundle exec ruby bin\newrelic_aws`) + +### Installation with Chef/Puppet + +[Chef](http://www.getchef.com) and [Puppet](http://puppetlabs.com) are tools that automate software installation. The Amazon CloudWatch plugin has installation support for both: + + - [Chef Cookbook](http://community.opscode.com/cookbooks/newrelic_plugins) + - [Puppet Module](https://forge.puppetlabs.com/newrelic/newrelic_plugins) + +**Note:** For more information on using Chef and Puppet with New Relic, see the New Relic [docs](https://docs.newrelic.com/docs/plugins/plugin-installation-with-chef-and-puppet). + +## Configuration + +This plugin is configured through the `config/newrelic_plugin.yml` file. It requires: + +- a New Relic license key that can be found at https://rpm.newrelic.com/extensions/com.newrelic.aws.ec2 +- an AWS Access Key +- an AWS Secret Key + +### Regions +The plugin can also be configured to query specific CloudWatch regions, e.g. `us-east-1` or `us-west-1`. By default the plugin will query all available regions. + +``` + regions: + - us-east-1 + - us-east-2 + - us-west-1 +``` + +### Amazon ElastiCache + +Amazon ElastiCache supports both Memcached and Redis caching technologies. The Memcached agent is configured under the `ec` section, while the Redis agent is configured under the `ecr` section. + +### Tag Filtering + +Filtering instances by tags is supported for both `EC2` and `EBS`. Details on adding tags to instances is available in the [AWS documentation](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). + +A list of case-sensitive tags can be added to the yml configuration for `EC2` and `EBS` and only instances containing one or more of those tags will be monitored. This can reduce CloudWatch requests and costs. + +Tagged instances will be monitored if they: + +- have a tag with a matching key in the tag list (e.g. tagged with key `newrelic_monitored` and any value: `true`, `yes`, etc.) +- have a `Name` or `name` tag with a matching value in the tag list (e.g. tagged with key `Name` and value `prod_1_db`) + +If there are no configured tags, all available instances will be monitored. This is the default behavior. + +``` +... +agents: + ec2: + enabled: true + tags: + - newrelic_monitored + - prod_1_db + ebs: + enabled: true + tags: + - newrelic_monitored +``` + +### RDS Instance Filtering +When an IAM policy for rds:DescribeDBInstances has Resource restrictions, the rds-describe-db-instances call will fail when an allowed DBInstanceIdentifier is not specified. + +If there are no configured instance_identifiers, all available instances will be monitored. This is the default behavior. + +``` +... +agents: + rds: + enabled: true + instance_identifiers: + - db1 + - db2 +``` + +### CloudWatch Delay +As noted below, there is a default 60 second delay in reporting metrics from CloudWatch which adjusts the time window for queried data. This is due to CloudWatch metrics not being immediately available for querying as they may take some time to process. Unfortunately there is little that can be done from the plugin to address this, besides adjusting the time window for metric querying from CloudWatch. The configuration option `cloudwatch_delay` can be specified for each AWS agent to override the default 60 second delay. + +``` +... +agents: + ec2: + enabled: true + cloudwatch_delay: 120 + ebs: + enabled: true +... +``` + +####Affected Metrics + +- `RDS/CPU Utilization` - If you see 0% CPU utilization, try increasing the `cloudwatch_delay` option for the AWS RDS agent. + +## AMI +This plugin is also available as an Amazon Machine Image (AMI) via the AWS Marketplace. Learn more, then quickly install and configure the AMI here: https://aws.amazon.com/marketplace/pp/B01KLLCQZE/ + +The AMI takes the contents of `config/newrelic_plugin.yml` as user-data, which is configured when creating the EC2 instance. Once the instance is running with valid user-data, no further action is required. To change the configuration, terminate the current instance and create another. + +If you like the AMI, please [leave a 5-star review](https://aws.amazon.com/marketplace/review/product-reviews/ref=dtl_pop_customer_reviews?ie=UTF8&asin=B01KLLCQZE) in the AWS Marketplace. + +If you don't like the AMI, New Relic would appreciate that you do not leave a bad review on the AWS Marketplace. Instead, open a ticket with [New Relic Support](https://support.newrelic.com) and let us know what we could do better. We take your feedback very seriously - and by opening a ticket, we can notify you when we've addressed your feedback. + +## IAM (AWS API Credentials) + +This plugin requires AWS API credentials, using IAM is highly recommended, giving it read-only access to select services. + +You will need to create a new IAM group, `NewRelicCloudWatch`, where the permissions will be defined. You will want to use a custom policy for the group, `NewRelicCloudWatch`, using the following JSON for the policy document. + +``` +{ + "Version": "2012-10-17", + "Statement": [ + { + "Action": [ + "autoscaling:Describe*", + "cloudwatch:Describe*", + "cloudwatch:List*", + "cloudwatch:Get*", + "ec2:Describe*", + "ec2:Get*", + "ec2:ReportInstanceStatus", + "elasticache:DescribeCacheClusters", + "elasticloadbalancing:Describe*", + "sqs:GetQueueAttributes", + "sqs:ListQueues", + "rds:DescribeDBInstances", + "SNS:ListTopics" + ], + "Effect": "Allow", + "Resource": "*" + } + ] +} +``` + +To get API credentials, a IAM user must be created, `NewRelicCloudWatch`. Be sure to save the user access key id and secret access key on creation. Add the user to the `NewRelicCloudWatch` IAM group. Use the IAM user API credentials in the plugin configuration file. + +## Notes +- CloudWatch detailed monitoring is recommended, please enable it when available. (see *Using Amazon CloudWatch* section on http://aws.amazon.com/cloudwatch/) +- Chart x-axis (time) is off by 60 seconds, this is due to CloudWatch's lag in reporting metrics. +- Latest data point is used to fill gaps in low resolution metrics. + +## Keep this process running +You can use services like these to manage this process. + +- [Upstart](http://upstart.ubuntu.com/) +- [Systemd](http://www.freedesktop.org/wiki/Software/systemd/) +- [Runit](http://smarden.org/runit/) +- [Monit](http://mmonit.com/monit/) + +## Or run it as a daemon +The provided daemon.rb file allows newrelic_aws to run as a daemon. + + chmod +x bin/daemon + bundle exec bin/daemon start + bundle exec bin/daemon stop + +## For support +Plugin support and troubleshooting assistance can be obtained by visiting [support.newrelic.com](https://support.newrelic.com) + +## Contributing + +You are welcome to send pull requests to us - however, by doing so you agree that you are granting New Relic a non-exclusive, non-revokable, no-cost license to use the code, algorithms, patents, and ideas in that code in our products if we so choose. You also agree the code is provided as-is and you provide no warranties as to its fitness or correctness for any purpose. + +## Credits +The New Relic AWS plugin was originally authored by [Sean Porter](https://github.com/portertech) and the team at [Heavy Water Operations](http://hw-ops.com/). Subsequent updates and support are provided by [New Relic](http://newrelic.com/platform). diff --git a/bin/daemon b/bin/daemon new file mode 100755 index 0000000..6d4042e --- /dev/null +++ b/bin/daemon @@ -0,0 +1,14 @@ +#!/usr/bin/env ruby + +# +# Start +# - bundle exec bin/daemon start +# +# Stop +# - bundle exec bin/daemon stop +# + +require 'daemons' + +pwd = File.join(File.dirname(File.expand_path(__FILE__)), "../") +Daemons.run('bin/newrelic_aws', :dir_mode => :normal, :dir => pwd) diff --git a/bin/newrelic_aws b/bin/newrelic_aws new file mode 100755 index 0000000..0366ed2 --- /dev/null +++ b/bin/newrelic_aws @@ -0,0 +1,18 @@ +#! /usr/bin/env ruby + +# +# This plugin includes several AWS agents, each pulling data from CloudWatch. +# +# Sean Porter (portertech@hw-ops.com) +# 2013-04-23 +# + +unless $:.include?(File.dirname(__FILE__) + '/../lib/') + $: << File.dirname(__FILE__) + '/../lib' +end + +Dir.chdir(File.join(File.dirname(__FILE__), "../")) + +STDOUT.sync = true + +require 'newrelic_aws' diff --git a/config/template_newrelic_plugin.yml b/config/template_newrelic_plugin.yml index b8a9632..409878e 100644 --- a/config/template_newrelic_plugin.yml +++ b/config/template_newrelic_plugin.yml @@ -1,6 +1,5 @@ -# Please make sure to update the license_key information with the license key for your New Relic -# account. -# +# Please make sure to update the license_key information with the +# license key for your New Relic account. # newrelic: # @@ -11,11 +10,54 @@ newrelic: # Set to '1' for verbose output, remove for normal output. # All output goes to stdout/stderr. # - verbose: 1 + # verbose: 1 +# +# AWS configuration. +# +aws: + # Update with you AWS account keys: + access_key: 'YOUR_AWS_ACCESS_KEY_HERE' + secret_key: 'YOUR_AWS_SECRET_KEY_HERE' + + #Use the following if you would like to configure AWS keys with ENV variables + #access_key: <%= ENV['AWS_ACCESS_KEY'] %> + #secret_key: <%= ENV['AWS_SECRET_ACCESS_KEY'] %> + + #Use the following line disable the key checks and enable usage of AWS instance metadata for setting keys + #use_aws_metadata: true + + # Specify AWS regions to query for metrics + #regions: + # - us-west-1 + # - us-east-2 + # -# Agent Configuration: +# Agent configuration. # agents: - # this is where configuration for agents belongs - example: - hertz: 1000 + # + # Enable/disable agents with the enabled attribute or by commenting out each agent. + # + # Tag and instance ID filtering are available and are documented at + # https://github.com/newrelic-platform/newrelic_aws_cloudwatch_plugin/blob/master/README.md#tag-filtering + # + # An optional cloudwatch_delay attribute can be added for each agent + # which adjusts the time window in seconds for CloudWatch querying. + # Some of the Cloudwatch metrics are not immediately available and may + # require a delay before being queried. + ec2: + enabled: true + ebs: + enabled: true + elb: + enabled: false + rds: + enabled: false + sqs: + enabled: false + sns: + enabled: false + ec: + enabled: false + ecr: + enabled: false diff --git a/lib/newrelic_aws.rb b/lib/newrelic_aws.rb new file mode 100644 index 0000000..68b1d0e --- /dev/null +++ b/lib/newrelic_aws.rb @@ -0,0 +1,225 @@ +require "rubygems" +require "bundler/setup" + +require "newrelic_plugin" +require "newrelic_aws/components" +require "newrelic_aws/collectors" +require 'newrelic_aws/version' + +# AWS SDK debug logging +if NewRelic::Plugin::Config.config.newrelic['verbose'].to_i > 1 + AWS.config( + :logger => NewRelic::PlatformLogger, + :log_level => :debug + ) +end + +module NewRelicAWS + AWS_REGIONS = %w[ + us-east-1 + us-west-1 + us-west-2 + eu-west-1 + ap-southeast-1 + ap-southeast-2 + ap-northeast-1 + ap-northeast-2 + sa-east-1 + eu-central-1 + ap-south-1 + ] + + def self.agent_options_exist?(agent_options) + # agent is commented out when options are nil + !agent_options.nil? + end + + def self.get_enabled_option(agent_options) + if agent_options['enabled'].nil? + true + else + agent_options['enabled'] + end + end + + def self.agent_enabled?(agent) + enabled = false + agent_options = NewRelic::Plugin::Config.config.agents[agent.to_s] + if NewRelicAWS::agent_options_exist?(agent_options) + enabled = NewRelicAWS::get_enabled_option(agent_options) + end + enabled + end + + module Base + class Agent < NewRelic::Plugin::Agent::Base + def agent_name + self.class.name.split("::")[-2].downcase + end + + def collector_class + Collectors.const_get(agent_name.upcase) + end + + def agent_options + return @agent_options if @agent_options + @agent_options = NewRelic::Plugin::Config.config.options + aws = @agent_options["aws"] + if aws.is_a?(Hash) + if aws["use_aws_metadata"] == true + @agent_options["aws"]["access_key"] = nil + @agent_options["aws"]["secret_key"] = nil + else + unless aws["access_key"].is_a?(String) && + aws["secret_key"].is_a?(String) + raise NewRelic::Plugin::BadConfig, "Missing or invalid AWS configuration." + end + end + else + raise NewRelic::Plugin::BadConfig, "Missing or invalid AWS configuration." + end + @agent_options + end + + def aws_regions + if agent_options["aws"]["regions"] + Array(agent_options["aws"]["regions"]) + else + AWS_REGIONS + end + end + + def setup_metrics + @collectors = [] + aws_regions.each do |region| + NewRelic::PlatformLogger.info("Creating a #{agent_name} metrics collector for region: #{region}") + @collectors << collector_class.new( + agent_options["aws"]["access_key"], + agent_options["aws"]["secret_key"], + region, + agent_options["agents"][agent_name] + ) + end + @components_collection = Components::Collection.new("com.newrelic.aws.#{agent_name}", NewRelicAWS::VERSION) + @context = NewRelic::Binding::Context.new(NewRelic::Plugin::Config.config.newrelic['license_key']) + @context.version = NewRelicAWS::VERSION + end + + def poll_cycle + request = @context.get_request + start_time = Time.now + NewRelic::PlatformLogger.debug("############## start poll_cycle ############") + metric_count = 0 + @collectors.each do |collector| + collector.collect.each do |component_name, metric_name, unit, value| + component = @context.get_component(component_name, @components_collection.guid) + @components_collection.report_metric(request, component, metric_name, unit, value) unless component.nil? + metric_count += 1 + end + end + NewRelic::PlatformLogger.debug("#{metric_count} metrics collected in #{Time.now - start_time} seconds") + NewRelic::PlatformLogger.debug("############## end poll_cycle ############") + request.deliver + end + end + end + + module EC2 + class Agent < Base::Agent + agent_guid "com.newrelic.aws.ec2" + agent_version NewRelicAWS::VERSION + agent_human_labels("EC2") { "EC2" } + end + end + + module EBS + class Agent < Base::Agent + agent_guid "com.newrelic.aws.ebs" + agent_version NewRelicAWS::VERSION + agent_human_labels("EBS") { "EBS" } + end + end + + module ALB + class Agent < Base::Agent + agent_guid "com.newrelic.aws.alb" + agent_version NewRelicAWS::VERSION + agent_human_labels("ALB") { "ALB" } + end + end + + module ELB + class Agent < Base::Agent + agent_guid "com.newrelic.aws.elb" + agent_version NewRelicAWS::VERSION + agent_human_labels("ELB") { "ELB" } + end + end + + module RDS + class Agent < Base::Agent + agent_guid "com.newrelic.aws.rds" + agent_version NewRelicAWS::VERSION + agent_human_labels("RDS") { "RDS" } + end + end + + module DDB + class Agent < Base::Agent + agent_guid "com.newrelic.aws.ddb" + agent_version NewRelicAWS::VERSION + agent_human_labels("DynamoDB") { "DynamoDB" } + end + end + + module SQS + class Agent < Base::Agent + agent_guid "com.newrelic.aws.sqs" + agent_version NewRelicAWS::VERSION + agent_human_labels("SQS") { "SQS" } + end + end + + module SNS + class Agent < Base::Agent + agent_guid "com.newrelic.aws.sns" + agent_version NewRelicAWS::VERSION + agent_human_labels("SNS") { "SNS" } + end + end + + module EC + class Agent < Base::Agent + agent_guid "com.newrelic.aws.ec" + agent_version NewRelicAWS::VERSION + agent_human_labels("ElastiCache") { "ElastiCache" } + end + end + + module ECR + class Agent < Base::Agent + agent_guid "com.newrelic.aws.ecr" + agent_version NewRelicAWS::VERSION + agent_human_labels("ElastiCacheRedis") { "ElastiCacheRedis" } + end + end + + # + # Register each agent with the component. + # + NewRelic::Plugin::Setup.install_agent :ec2, EC2 if NewRelicAWS::agent_enabled?(:ec2) + NewRelic::Plugin::Setup.install_agent :ebs, EBS if NewRelicAWS::agent_enabled?(:ebs) + NewRelic::Plugin::Setup.install_agent :elb, ELB if NewRelicAWS::agent_enabled?(:elb) + NewRelic::Plugin::Setup.install_agent :alb, ALB if NewRelicAWS::agent_enabled?(:alb) + NewRelic::Plugin::Setup.install_agent :rds, RDS if NewRelicAWS::agent_enabled?(:rds) + # NewRelic::Plugin::Setup.install_agent :ddb, DDB # WIP + NewRelic::Plugin::Setup.install_agent :sqs, SQS if NewRelicAWS::agent_enabled?(:sqs) + NewRelic::Plugin::Setup.install_agent :sns, SNS if NewRelicAWS::agent_enabled?(:sns) + NewRelic::Plugin::Setup.install_agent :ec, EC if NewRelicAWS::agent_enabled?(:ec) + NewRelic::Plugin::Setup.install_agent :ecr, ECR if NewRelicAWS::agent_enabled?(:ecr) + + # + # Launch the agents; this never returns. + # + NewRelic::Plugin::Run.setup_and_run +end diff --git a/lib/newrelic_aws/collectors.rb b/lib/newrelic_aws/collectors.rb new file mode 100644 index 0000000..d6cbdcf --- /dev/null +++ b/lib/newrelic_aws/collectors.rb @@ -0,0 +1,14 @@ +require "time" +require "aws-sdk" + +require "newrelic_aws/collectors/base" +require "newrelic_aws/collectors/ec2" +require "newrelic_aws/collectors/ebs" +require "newrelic_aws/collectors/alb" +require "newrelic_aws/collectors/elb" +require "newrelic_aws/collectors/rds" +require "newrelic_aws/collectors/ddb" +require "newrelic_aws/collectors/sqs" +require "newrelic_aws/collectors/sns" +require "newrelic_aws/collectors/ec" +require "newrelic_aws/collectors/ecr" diff --git a/lib/newrelic_aws/collectors/alb.rb b/lib/newrelic_aws/collectors/alb.rb new file mode 100644 index 0000000..3a1c4a1 --- /dev/null +++ b/lib/newrelic_aws/collectors/alb.rb @@ -0,0 +1,61 @@ +module NewRelicAWS + module Collectors + class ALB < Base + def load_balancers + alb = AWS::ELB.new( + :access_key_id => @aws_access_key, + :secret_access_key => @aws_secret_key, + :region => @aws_region + ) + alb.load_balancers.map { |load_balancer| load_balancer.name } + end + + def metric_list + [ + ["ActiveConnectionCount", "Average", "Count", 0], + ["ClientTLSNegotiationErrorCount", "Sum", "Count", 0], + ["ConsumedLBCapacityUnits", "Sum", "Count", 0], + ["HealthyHostCount", "Maximum", "Count", 0], + ["HTTPCode_ELB_4XX", "Sum", "Count", 0], + ["HTTPCode_ELB_5XX", "Sum", "Count", 0], + ["HTTPCode_Target_2XX_Count", "Sum", "Count", 0], + ["HTTPCode_Target_3XX_Count", "Sum", "Count", 0], + ["HTTPCode_Target_4XX_Count", "Sum", "Count", 0], + ["HTTPCode_Target_5XX_Count", "Sum", "Count", 0], + ["NewConnectionCount", "Sum", "Count", 0], + ["ProcessedBytes", "Average", "Bytes", 0], + ["RequestCount", "Sum", "Count", 0], + ["RejectedConnectionCount", "Sum", "Count", 0], + ["TargetConnectionErrorCount", "Sum", "Count", 0], + ["TargetResponseTime", "Average", "Milliseconds", 0], + ["TargetTLSNegotiationErrorCount", "Sum", "Count", 0], + ["UnHealthyHostCount", "Maximum", "Count", 0], + ] + end + + def collect + data_points = [] + load_balancers.each do |load_balancer_name| + metric_list.each do |(metric_name, statistic, unit, default_value)| + data_point = get_data_point( + :namespace => "AWS/ApplicationELB", + :metric_name => metric_name, + :statistic => statistic, + :unit => unit, + :default_value => default_value, + :dimension => { + :name => "LoadBalancerName", + :value => load_balancer_name + } + ) + NewRelic::PlatformLogger.debug("metric_name: #{metric_name}, statistic: #{statistic}, unit: #{unit}, response: #{data_point.inspect}") + unless data_point.nil? + data_points << data_point + end + end + end + data_points + end + end + end +end diff --git a/lib/newrelic_aws/collectors/base.rb b/lib/newrelic_aws/collectors/base.rb new file mode 100644 index 0000000..bbf1822 --- /dev/null +++ b/lib/newrelic_aws/collectors/base.rb @@ -0,0 +1,70 @@ +module NewRelicAWS + module Collectors + class Base + def initialize(access_key, secret_key, region, options) + @aws_access_key = access_key + @aws_secret_key = secret_key + @aws_region = region + @cloudwatch = AWS::CloudWatch.new( + :access_key_id => @aws_access_key, + :secret_access_key => @aws_secret_key, + :region => @aws_region + ) + @cloudwatch_delay = options[:cloudwatch_delay] || 60 + end + + def get_data_point(options) + options[:period] ||= 60 + options[:start_time] ||= (Time.now.utc - (@cloudwatch_delay + options[:period])).iso8601 + options[:end_time] ||= (Time.now.utc - @cloudwatch_delay).iso8601 + options[:dimensions] ||= [options[:dimension]] + NewRelic::PlatformLogger.info("Retrieving statistics: " + options.inspect) + begin + statistics = @cloudwatch.client.get_metric_statistics( + :namespace => options[:namespace], + :metric_name => options[:metric_name], + :unit => options[:unit], + :statistics => [options[:statistic]], + :period => options[:period], + :start_time => options[:start_time], + :end_time => options[:end_time], + :dimensions => options[:dimensions] + ) + rescue => error + NewRelic::PlatformLogger.error("Unexpected error: " + error.message) + NewRelic::PlatformLogger.debug("Backtrace: " + error.backtrace.join("\n ")) + raise error + end + NewRelic::PlatformLogger.info("Retrieved statistics: #{statistics.inspect}") + + point = statistics[:datapoints].last + value = get_value(point, options) + return if value.nil? + + component_name = get_component_name(options) + [component_name, options[:metric_name], options[:unit].downcase, value] + end + + def collect + [] + end + + private + + def get_value(point, options) + value = options[:default_value] + unless point.nil? + statistic = options[:statistic].downcase.to_sym + value = point[statistic] + end + value + end + + def get_component_name(options) + component_name = options[:component_name] + component_name ||= options[:dimensions].map { |dimension| dimension[:value] }.join("/") + component_name + end + end + end +end diff --git a/lib/newrelic_aws/collectors/ddb.rb b/lib/newrelic_aws/collectors/ddb.rb new file mode 100644 index 0000000..8904d34 --- /dev/null +++ b/lib/newrelic_aws/collectors/ddb.rb @@ -0,0 +1,49 @@ +module NewRelicAWS + module Collectors + class DDB < Base + def tables + ddb = AWS::DynamoDB.new( + :access_key_id => @aws_access_key, + :secret_access_key => @aws_secret_key + ) + ddb.tables.map { |table| table.name } + end + + def metric_list + [ + ["UserErrors", "Sum", "Count"], + ["SystemErrors", "Sum", "Count"], + ["ThrottledRequests", "Sum", "Count"], + ["ProvisionedReadCapacityUnits", "Sum", "Count"], + ["ProvisionedWriteCapacityUnits", "Sum", "Count"], + ["ConsumedReadCapacityUnits", "Sum", "Count"], + ["ConsumedWriteCapacityUnits", "Sum", "Count"], + ["ReturnedItemCount", "Sum", "Count"] + ] + end + + def collect + data_points = [] + tables.each do |table_name| + metric_list.each do |(metric_name, statistic, unit)| + data_point = get_data_point( + :namespace => "AWS/DynamoDB", + :metric_name => metric_name, + :statistic => statistic, + :unit => unit, + :dimension => { + :name => "TableName", + :value => table_name + } + ) + NewRelic::PlatformLogger.debug("metric_name: #{metric_name}, statistic: #{statistic}, unit: #{unit}, response: #{data_point.inspect}") + unless data_point.nil? + data_points << data_point + end + end + end + data_points + end + end + end +end diff --git a/lib/newrelic_aws/collectors/ebs.rb b/lib/newrelic_aws/collectors/ebs.rb new file mode 100644 index 0000000..3e5a450 --- /dev/null +++ b/lib/newrelic_aws/collectors/ebs.rb @@ -0,0 +1,76 @@ +module NewRelicAWS + module Collectors + class EBS < Base + def initialize(access_key, secret_key, region, options) + super(access_key, secret_key, region, options) + @ec2 = AWS::EC2.new( + :access_key_id => @aws_access_key, + :secret_access_key => @aws_secret_key, + :region => @aws_region + ) + @tags = options[:tags] + end + + def volumes + if @tags + tagged_volumes + else + @ec2.volumes.filter('status', 'in-use') + end + end + + def tagged_volumes + volumes = @ec2.volumes.filter('status', 'in-use').tagged(@tags).to_a + volumes.concat(@ec2.volumes.filter('status', 'in-use').tagged('Name', 'name').tagged_values(@tags).to_a) + volumes + end + + def metric_list + [ + ["VolumeReadBytes", "Sum", "Bytes"], + ["VolumeWriteBytes", "Sum", "Bytes"], + ["VolumeReadOps", "Sum", "Count"], + ["VolumeWriteOps", "Sum", "Count"], + ["VolumeTotalReadTime", "Sum", "Seconds"], + ["VolumeTotalWriteTime", "Sum", "Seconds"], + ["VolumeIdleTime", "Sum", "Seconds"], + ["VolumeQueueLength", "Sum", "Count"], + ["VolumeThroughputPercentage", "Average", "Percent"], + ["VolumeConsumedReadWriteOps", "Sum", "Count"] + ] + end + + def collect + data_points = [] + volumes.each do |volume| + detailed = !!volume.iops + name_tag = volume.tags.detect { |tag| tag.first =~ /^name$/i } + metric_list.each do |(metric_name, statistic, unit)| + period = detailed ? 60 : 300 + time_offset = detailed ? 60 : 600 + time_offset += @cloudwatch_delay + data_point = get_data_point( + :namespace => "AWS/EBS", + :metric_name => metric_name, + :statistic => statistic, + :unit => unit, + :dimension => { + :name => "VolumeId", + :value => volume.id + }, + :period => period, + :start_time => (Time.now.utc - (time_offset + period)).iso8601, + :end_time => (Time.now.utc - time_offset).iso8601, + :component_name => name_tag.nil? ? volume.id : "#{name_tag.last} (#{volume.id})" + ) + NewRelic::PlatformLogger.debug("metric_name: #{metric_name}, statistic: #{statistic}, unit: #{unit}, response: #{data_point.inspect}") + unless data_point.nil? + data_points << data_point + end + end + end + data_points + end + end + end +end diff --git a/lib/newrelic_aws/collectors/ec.rb b/lib/newrelic_aws/collectors/ec.rb new file mode 100644 index 0000000..d94618a --- /dev/null +++ b/lib/newrelic_aws/collectors/ec.rb @@ -0,0 +1,98 @@ +module NewRelicAWS + module Collectors + class EC < Base + def clusters(engine = 'memcached') + ec = AWS::ElastiCache.new( + :access_key_id => @aws_access_key, + :secret_access_key => @aws_secret_key, + :region => @aws_region + ) + clusters = ec.client.describe_cache_clusters(:show_cache_node_info => true) + clusters[:cache_clusters].map do |cluster| + if cluster[:engine] == engine + { + :id => cluster[:cache_cluster_id], + :nodes => cluster[:cache_nodes].map { |node| node[:cache_node_id] } + } + end + end.compact + end + + def metric_list + [ + ["CPUUtilization", "Average", "Percent"], + ["SwapUsage", "Average", "Bytes"], + ["FreeableMemory", "Average", "Bytes"], + ["NetworkBytesIn", "Average", "Bytes"], + ["NetworkBytesOut", "Average", "Bytes"], + ["BytesUsedForCacheItems", "Average", "Bytes"], + ["BytesReadIntoMemcached", "Sum", "Bytes"], + ["BytesWrittenOutFromMemcached", "Sum", "Bytes"], + ["CasBadval", "Sum", "Count"], + ["CasHits", "Sum", "Count"], + ["CasMisses", "Sum", "Count"], + ["CmdFlush", "Sum", "Count"], + ["CmdGet", "Sum", "Count"], + ["CmdSet", "Sum", "Count"], + ["CurrConnections", "Average", "Count"], + ["CurrItems", "Average", "Count"], + ["DecrHits", "Sum", "Count"], + ["DecrMisses", "Sum", "Count"], + ["DeleteHits", "Sum", "Count"], + ["DeleteMisses", "Sum", "Count"], + ["Evictions", "Sum", "Count"], + ["GetHits", "Sum", "Count"], + ["GetMisses", "Sum", "Count"], + ["IncrHits", "Sum", "Count"], + ["IncrMisses", "Sum", "Count"], + ["Reclaimed", "Sum", "Count"], + ["BytesUsedForHash", "Average", "Bytes"], + ["CmdConfigGet", "Sum", "Count"], + ["CmdConfigSet", "Sum", "Count"], + ["CmdTouch", "Sum", "Count"], + ["CurrConfig", "Average", "Count"], + ["EvictedUnfetched", "Sum", "Count"], + ["ExpiredUnfetched", "Sum", "Count"], + ["SlabsMoved", "Sum", "Count"], + ["TouchHits", "Sum", "Count"], + ["TouchMisses", "Sum", "Count"], + ["NewConnections", "Sum", "Count"], + ["NewItems", "Sum", "Count"], + ["UnusedMemory", "Average", "Bytes"] + ] + end + + def collect + data_points = [] + clusters.each do |cluster| + metric_list.each do |(metric_name, statistic, unit, default_value)| + cluster[:nodes].each do |node_id| + data_point = get_data_point( + :namespace => "AWS/ElastiCache", + :metric_name => metric_name, + :statistic => statistic, + :unit => unit, + :default_value => default_value, + :dimensions => [ + { + :name => "CacheClusterId", + :value => cluster[:id] + }, + { + :name => "CacheNodeId", + :value => node_id + } + ] + ) + NewRelic::PlatformLogger.debug("metric_name: #{metric_name}, statistic: #{statistic}, unit: #{unit}, response: #{data_point.inspect}") + unless data_point.nil? + data_points << data_point + end + end + end + end + data_points + end + end + end +end diff --git a/lib/newrelic_aws/collectors/ec2.rb b/lib/newrelic_aws/collectors/ec2.rb new file mode 100644 index 0000000..3604c69 --- /dev/null +++ b/lib/newrelic_aws/collectors/ec2.rb @@ -0,0 +1,72 @@ +module NewRelicAWS + module Collectors + class EC2 < Base + def initialize(access_key, secret_key, region, options) + super(access_key, secret_key, region, options) + @ec2 = AWS::EC2.new( + :access_key_id => @aws_access_key, + :secret_access_key => @aws_secret_key, + :region => @aws_region + ) + @tags = options[:tags] + end + + def instances + if @tags + tagged_instances + else + @ec2.instances + end + end + + def tagged_instances + instances = @ec2.instances.tagged(@tags).to_a + instances.concat(@ec2.instances.tagged('Name', 'name').tagged_values(@tags).to_a) + instances + end + + def metric_list + [ + ["CPUUtilization", "Average", "Percent"], + ["DiskReadOps", "Sum", "Count"], + ["DiskWriteOps", "Sum", "Count"], + ["DiskWriteBytes" , "Sum", "Bytes"], + ["NetworkIn", "Sum", "Bytes"], + ["NetworkOut", "Sum", "Bytes"] + ] + end + + def collect + data_points = [] + instances.each do |instance| + detailed = instance.monitoring == :enabled + name_tag = instance.tags.detect { |tag| tag.first =~ /^name$/i } + metric_list.each do |(metric_name, statistic, unit)| + period = detailed ? 60 : 300 + time_offset = detailed ? 60 : 600 + time_offset += @cloudwatch_delay + data_point = get_data_point( + :namespace => "AWS/EC2", + :metric_name => metric_name, + :statistic => statistic, + :unit => unit, + :dimension => { + :name => "InstanceId", + :value => instance.id + }, + :period => period, + :start_time => (Time.now.utc - (time_offset + period)).iso8601, + :end_time => (Time.now.utc - time_offset).iso8601, + :component_name => name_tag.nil? ? instance.id : "#{name_tag.last} (#{instance.id})" + ) + NewRelic::PlatformLogger.debug("metric_name: #{metric_name}, statistic: #{statistic}, unit: #{unit}, response: #{data_point.inspect}") + unless data_point.nil? + data_points << data_point + end + end + end + data_points + end + end + end +end diff --git a/lib/newrelic_aws/collectors/ecr.rb b/lib/newrelic_aws/collectors/ecr.rb new file mode 100644 index 0000000..509ffa6 --- /dev/null +++ b/lib/newrelic_aws/collectors/ecr.rb @@ -0,0 +1,36 @@ +module NewRelicAWS + module Collectors + class ECR < EC + def clusters + super('redis') + end + + def metric_list + [ + ["CPUUtilization", "Average", "Percent"], + ["SwapUsage", "Average", "Bytes"], + ["FreeableMemory", "Average", "Bytes"], + ["NetworkBytesIn", "Average", "Bytes", 0], + ["NetworkBytesOut", "Average", "Bytes", 0], + ["CurrConnections", "Average", "Count"], + ["Evictions", "Sum", "Count"], + ["Reclaimed", "Sum", "Count"], + ["NewConnections", "Sum", "Count"], + ["BytesUsedForCache", "Average", "Bytes"], + ["CacheHits", "Sum", "Count"], + ["CacheMisses", "Sum", "Count"], + ["GetTypeCmds", "Sum", "Count", 0], + ["SetTypeCmds", "Sum", "Count", 0], + ["KeyBasedCmds", "Sum", "Count", 0], + ["StringBasedCmds", "Sum", "Count"], + ["HashBasedCmds", "Sum", "Count", 0], + ["ListBasedCmds", "Sum", "Count", 0], + ["SetBasedCmds", "Sum", "Count", 0], + ["SortedSetBasedCmds", "Sum", "Count", 0], + ["CurrItems", "Average", "Count"] + ] + end + + end + end +end \ No newline at end of file diff --git a/lib/newrelic_aws/collectors/elb.rb b/lib/newrelic_aws/collectors/elb.rb new file mode 100644 index 0000000..e68effc --- /dev/null +++ b/lib/newrelic_aws/collectors/elb.rb @@ -0,0 +1,56 @@ +module NewRelicAWS + module Collectors + class ELB < Base + def load_balancers + elb = AWS::ELB.new( + :access_key_id => @aws_access_key, + :secret_access_key => @aws_secret_key, + :region => @aws_region + ) + elb.load_balancers.map { |load_balancer| load_balancer.name } + end + + def metric_list + [ + ["Latency", "Average", "Seconds"], + ["RequestCount", "Sum", "Count", 0], + ["HealthyHostCount", "Maximum", "Count", 0], + ["UnHealthyHostCount", "Maximum", "Count", 0], + ["HTTPCode_ELB_4XX", "Sum", "Count", 0], + ["HTTPCode_ELB_5XX", "Sum", "Count", 0], + ["HTTPCode_Backend_2XX", "Sum", "Count", 0], + ["HTTPCode_Backend_3XX", "Sum", "Count", 0], + ["HTTPCode_Backend_4XX", "Sum", "Count", 0], + ["HTTPCode_Backend_5XX", "Sum", "Count", 0], + ["BackendConnectionErrors", "Sum", "Count", 0], + ["SurgeQueueLength", "Maximum", "Count", 0], + ["SpilloverCount", "Sum", "Count", 0] + ] + end + + def collect + data_points = [] + load_balancers.each do |load_balancer_name| + metric_list.each do |(metric_name, statistic, unit, default_value)| + data_point = get_data_point( + :namespace => "AWS/ELB", + :metric_name => metric_name, + :statistic => statistic, + :unit => unit, + :default_value => default_value, + :dimension => { + :name => "LoadBalancerName", + :value => load_balancer_name + } + ) + NewRelic::PlatformLogger.debug("metric_name: #{metric_name}, statistic: #{statistic}, unit: #{unit}, response: #{data_point.inspect}") + unless data_point.nil? + data_points << data_point + end + end + end + data_points + end + end + end +end diff --git a/lib/newrelic_aws/collectors/rds.rb b/lib/newrelic_aws/collectors/rds.rb new file mode 100644 index 0000000..7fdbfa9 --- /dev/null +++ b/lib/newrelic_aws/collectors/rds.rb @@ -0,0 +1,62 @@ +module NewRelicAWS + module Collectors + class RDS < Base + def initialize(access_key, secret_key, region, options) + super(access_key, secret_key, region, options) + @instance_ids = options[:instance_identifiers] + end + + def instance_ids + return @instance_ids if @instance_ids + rds = AWS::RDS.new( + :access_key_id => @aws_access_key, + :secret_access_key => @aws_secret_key, + :region => @aws_region + ) + rds.instances.map { |instance| instance.id } + end + + def metric_list + [ + ["BinLogDiskUsage", "Average", "Bytes"], + ["CPUUtilization", "Average", "Percent"], + ["DatabaseConnections", "Average", "Count"], + ["DiskQueueDepth", "Average", "Count"], + ["FreeableMemory", "Average", "Bytes"], + ["FreeStorageSpace", "Average", "Bytes"], + ["ReplicaLag", "Average", "Seconds"], + ["SwapUsage", "Average", "Bytes"], + ["ReadIOPS", "Average", "Count/Second"], + ["WriteIOPS", "Average", "Count/Second"], + ["ReadLatency", "Average", "Seconds"], + ["WriteLatency", "Average", "Seconds"], + ["ReadThroughput", "Average", "Bytes/Second"], + ["WriteThroughput", "Average", "Bytes/Second"] + ] + end + + def collect + data_points = [] + instance_ids.each do |instance_id| + metric_list.each do |(metric_name, statistic, unit)| + data_point = get_data_point( + :namespace => "AWS/RDS", + :metric_name => metric_name, + :statistic => statistic, + :unit => unit, + :dimension => { + :name => "DBInstanceIdentifier", + :value => instance_id + } + ) + NewRelic::PlatformLogger.debug("metric_name: #{metric_name}, statistic: #{statistic}, unit: #{unit}, response: #{data_point.inspect}") + unless data_point.nil? + data_points << data_point + end + end + end + data_points + end + end + end +end diff --git a/lib/newrelic_aws/collectors/sns.rb b/lib/newrelic_aws/collectors/sns.rb new file mode 100644 index 0000000..bb4fef9 --- /dev/null +++ b/lib/newrelic_aws/collectors/sns.rb @@ -0,0 +1,51 @@ +module NewRelicAWS + module Collectors + class SNS < Base + def topic_names + sns = AWS::SNS.new( + :access_key_id => @aws_access_key, + :secret_access_key => @aws_secret_key, + :region => @aws_region + ) + sns.topics.map { |topic| topic.name } + end + + def metric_list + [ + ["NumberOfMessagesPublished", "Sum", "Count"], + ["PublishSize", "Average", "Bytes"], + ["NumberOfNotificationsDelivered" , "Sum", "Count"], + ["NumberOfNotificationsFailed", "Sum", "Count"] + ] + end + + def collect + data_points = [] + topic_names.each do |topic_name| + metric_list.each do |(metric_name, statistic, unit)| + period = 300 + time_offset = 600 + @cloudwatch_delay + data_point = get_data_point( + :namespace => "AWS/SNS", + :metric_name => metric_name, + :statistic => statistic, + :unit => unit, + :dimension => { + :name => "TopicName", + :value => topic_name + }, + :period => period, + :start_time => (Time.now.utc - (time_offset + period)).iso8601, + :end_time => (Time.now.utc - time_offset).iso8601 + ) + NewRelic::PlatformLogger.debug("metric_name: #{metric_name}, statistic: #{statistic}, unit: #{unit}, response: #{data_point.inspect}") + unless data_point.nil? + data_points << data_point + end + end + end + data_points + end + end + end +end diff --git a/lib/newrelic_aws/collectors/sqs.rb b/lib/newrelic_aws/collectors/sqs.rb new file mode 100644 index 0000000..9923ad1 --- /dev/null +++ b/lib/newrelic_aws/collectors/sqs.rb @@ -0,0 +1,55 @@ +module NewRelicAWS + module Collectors + class SQS < Base + def queue_urls + sqs = AWS::SQS.new( + :access_key_id => @aws_access_key, + :secret_access_key => @aws_secret_key, + :region => @aws_region + ) + sqs.queues.map { |queue| queue.url } + end + + def metric_list + [ + ["NumberOfMessagesSent", "Sum", "Count"], + ["SentMessageSize", "Average", "Bytes"], + ["NumberOfMessagesReceived", "Sum", "Count"], + ["NumberOfEmptyReceives", "Sum", "Count"], + ["NumberOfMessagesDeleted", "Sum", "Count"], + ["ApproximateNumberOfMessagesDelayed", "Average", "Count"], + ["ApproximateNumberOfMessagesVisible", "Average", "Count"], + ["ApproximateNumberOfMessagesNotVisible", "Average", "Count"] + ] + end + + def collect + data_points = [] + queue_urls.each do |url| + metric_list.each do |(metric_name, statistic, unit)| + period = 300 + time_offset = 600 + @cloudwatch_delay + data_point = get_data_point( + :namespace => "AWS/SQS", + :metric_name => metric_name, + :statistic => statistic, + :unit => unit, + :dimension => { + :name => "QueueName", + :value => url.split("/").last + }, + :period => period, + :start_time => (Time.now.utc - (time_offset + period)).iso8601, + :end_time => (Time.now.utc - time_offset).iso8601 + ) + NewRelic::PlatformLogger.debug("metric_name: #{metric_name}, statistic: #{statistic}, unit: #{unit}, response: #{data_point.inspect}") + unless data_point.nil? + data_points << data_point + end + end + end + data_points + end + end + end +end diff --git a/lib/newrelic_aws/components.rb b/lib/newrelic_aws/components.rb new file mode 100644 index 0000000..d8e4af4 --- /dev/null +++ b/lib/newrelic_aws/components.rb @@ -0,0 +1,14 @@ +module NewRelicAWS + module Components + class Collection + attr_reader :guid + def initialize(guid, version) + @guid = guid + end + + def report_metric(request, component, metric_name, unit, value) + request.add_metric(component, "Component/#{metric_name}[#{unit}]", value) + end + end + end +end diff --git a/lib/newrelic_aws/version.rb b/lib/newrelic_aws/version.rb new file mode 100644 index 0000000..f13785f --- /dev/null +++ b/lib/newrelic_aws/version.rb @@ -0,0 +1,3 @@ +module NewRelicAWS + VERSION = '3.3.5' +end diff --git a/newrelic_example_agent b/newrelic_example_agent deleted file mode 100755 index 4e92a46..0000000 --- a/newrelic_example_agent +++ /dev/null @@ -1,48 +0,0 @@ -#! /usr/bin/env ruby - -# -# This is an example agent which generates synthetic data. -# A 1mHz (one cycle every 16 minutes) sin+1, cos+1 and sin+5 wave is generated, -# using the Unix epoch as the base. -# -# Robert R. Henry (rrh@newrelic.com) -# 2012-10-18 -# - -require "rubygems" -require "bundler/setup" - -require "newrelic_plugin" - -module ExampleAgent - - class Agent < NewRelic::Plugin::Agent::Base - - agent_guid "_DROP_GUID_FROM_PLUGIN_HERE_" - agent_version "0.0.3" - agent_config_options :hertz # frequency of the periodic functions - agent_human_labels("Example Agent") { "Synthetic example data" } - - def poll_cycle - x = Time.now.to_f * hertz * Math::PI * 2 - report_metric "SIN", "Value", Math.sin(x) + 1.0 - report_metric "COS", "Value", Math.cos(x) + 1.0 - report_metric "BIASSIN", "Value", Math.sin(x) + 5.0 - end - - end - - # - # Register this agent with the component. - # The ExampleAgent is the name of the module that defines this - # driver (the module must contain at least three classes - a - # PollCycle, a Metric and an Agent class, as defined above). - # - NewRelic::Plugin::Setup.install_agent :example, ExampleAgent - - # - # Launch the agent; this never returns. - # - NewRelic::Plugin::Run.setup_and_run - -end