From 72c61694d5dd3b4649dfd71ee1b63c8b86e1f0dd Mon Sep 17 00:00:00 2001
From: Anthony Johnson <ansoni@gmail.com>
Date: Wed, 25 Feb 2015 20:59:29 -0500
Subject: [PATCH 1/8] first pass for credits
---
.../ice/basic/BasicLineItemProcessor.java | 85 ++++++++++++++++---
1 file changed, 74 insertions(+), 11 deletions(-)
diff --git a/src/java/com/netflix/ice/basic/BasicLineItemProcessor.java b/src/java/com/netflix/ice/basic/BasicLineItemProcessor.java
index f16a88dd..739cdf21 100644
--- a/src/java/com/netflix/ice/basic/BasicLineItemProcessor.java
+++ b/src/java/com/netflix/ice/basic/BasicLineItemProcessor.java
@@ -28,6 +28,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.Arrays;
import java.util.List;
import java.util.Map;
@@ -88,23 +89,73 @@ public long getEndMillis(String[] items) {
}
public Result process(long startMilli, boolean processDelayed, ProcessorConfig config, String[] items, Map<Product, ReadWriteData> usageDataByProduct, Map<Product, ReadWriteData> costDataByProduct, Map<String, Double> ondemandRate) {
- if (StringUtils.isEmpty(items[accountIdIndex]) ||
- StringUtils.isEmpty(items[productIndex]) ||
- StringUtils.isEmpty(items[usageTypeIndex]) ||
- StringUtils.isEmpty(items[operationIndex]) ||
- StringUtils.isEmpty(items[usageQuantityIndex]) ||
- StringUtils.isEmpty(items[costIndex]))
+
+ if (StringUtils.isEmpty(items[costIndex])) {
+ logger.debug("Ignoring Record due to missing Cost - " + Arrays.toString(items));
+ return Result.ignore;
+ }
+
+ double costValue = Double.parseDouble(items[costIndex]);
+ boolean credit = false;
+
+ // make sure we don't ignore credits
+ if (costValue < 0) {
+ credit = true;
+ }
+
+ //fail-fast on records we can't process
+ if (StringUtils.isEmpty(items[accountIdIndex])) {
+ logger.debug("Ignoring Record due to missing Account Id - " + Arrays.toString(items));
+ return Result.ignore;
+ } else if (StringUtils.isEmpty(items[productIndex])) {
+ logger.debug("Ignoring Record due to missing Product - " + Arrays.toString(items));
return Result.ignore;
+ }
+
+ // check other records. We might have to help some credit rows to show up properly
+ if (StringUtils.isEmpty(items[usageTypeIndex])) {
+ if (credit) {
+ items[usageTypeIndex]="credit";
+ } else {
+ logger.debug("Ignoring Record due to missing Usage Type - " + Arrays.toString(items));
+ return Result.ignore;
+ }
+ }
+ if (StringUtils.isEmpty(items[operationIndex])) {
+ if (credit) {
+ items[operationIndex]="credit";
+ } else {
+ logger.debug("Ignoring Record due to missing Operation - " + Arrays.toString(items));
+ return Result.ignore;
+ }
+ }
+ if (StringUtils.isEmpty(items[usageQuantityIndex])) {
+ if (credit) {
+ items[usageQuantityIndex]="1";
+ } else {
+ logger.debug("Ignoring Record due to missing Usage Quantity - " + Arrays.toString(items));
+ return Result.ignore;
+ }
+ }
Account account = config.accountService.getAccountById(items[accountIdIndex]);
if (account == null)
return Result.ignore;
double usageValue = Double.parseDouble(items[usageQuantityIndex]);
- double costValue = Double.parseDouble(items[costIndex]);
long millisStart;
long millisEnd;
+
+ boolean monthlyCredit=false;
+ Result result = Result.hourly;
+ if (credit && items[startTimeIndex].isEmpty()) {
+ items[startTimeIndex]=new DateTime(startMilli, DateTimeZone.UTC).toString(amazonBillingDateFormat);
+ int numHoursInMonth = new DateTime(startMilli, DateTimeZone.UTC).dayOfMonth().getMaximumValue() * 24;
+ items[endTimeIndex]=new DateTime(startMilli, DateTimeZone.UTC).plusHours(numHoursInMonth).toString(amazonBillingDateFormat);
+ monthlyCredit=true;
+ }
+
try {
millisStart = amazonBillingDateFormat.parseMillis(items[startTimeIndex]);
millisEnd = amazonBillingDateFormat.parseMillis(items[endTimeIndex]);
@@ -116,7 +167,7 @@ public Result process(long startMilli, boolean processDelayed, ProcessorConfig c
Product product = config.productService.getProductByAwsName(items[productIndex]);
boolean reservationUsage = "Y".equals(items[reservedIndex]);
- ReformedMetaData reformedMetaData = reform(millisStart, config, product, reservationUsage, items[operationIndex], items[usageTypeIndex], items[descriptionIndex], costValue);
+ ReformedMetaData reformedMetaData = reform(millisStart, config, product, reservationUsage, items[operationIndex], items[usageTypeIndex], items[descriptionIndex], costValue, credit);
product = reformedMetaData.product;
Operation operation = reformedMetaData.operation;
UsageType usageType = reformedMetaData.usageType;
@@ -125,7 +176,6 @@ public Result process(long startMilli, boolean processDelayed, ProcessorConfig c
int startIndex = (int)((millisStart - startMilli)/ AwsUtils.hourMillis);
int endIndex = (int)((millisEnd + 1000 - startMilli)/ AwsUtils.hourMillis);
- Result result = Result.hourly;
if (product == Product.ec2_instance) {
result = processEc2Instance(processDelayed, reservationUsage, operation, zone);
}
@@ -145,13 +195,18 @@ else if (product == Product.rds) {
result = processRds(usageType);
}
+ // make sure credits for the month are distributed across month
+
if (result == Result.ignore || result == Result.delay)
return result;
if (usageType.name.startsWith("TimedStorage-ByteHrs"))
result = Result.daily;
- boolean monthlyCost = StringUtils.isEmpty(items[descriptionIndex]) ? false : items[descriptionIndex].toLowerCase().contains("-month");
+ if (monthlyCredit)
+ result = Result.monthly;
+
+ boolean monthlyCost = StringUtils.isEmpty(items[descriptionIndex]) ? false : ( items[descriptionIndex].toLowerCase().contains("-month") || monthlyCredit );
ReadWriteData usageData = usageDataByProduct.get(null);
ReadWriteData costData = costDataByProduct.get(null);
@@ -167,6 +222,8 @@ else if (product == Product.rds) {
else if (result == Result.monthly) {
startIndex = 0;
endIndex = usageData.getNum();
+ logger.debug("Start Index: " + startIndex);
+ logger.debug("End Index: " + endIndex);
int numHoursInMonth = new DateTime(startMilli, DateTimeZone.UTC).dayOfMonth().getMaximumValue() * 24;
usageValue = usageValue * endIndex / numHoursInMonth;
costValue = costValue * endIndex / numHoursInMonth;
@@ -234,6 +291,9 @@ else if (result == Result.monthly) {
return result;
for (int i : indexes) {
+ if (credit) {
+ logger.debug("Index: " + i);
+ }
if (config.randomizer != null) {
@@ -345,7 +405,7 @@ private Result processRds(UsageType usageType) {
return Result.hourly;
}
- protected ReformedMetaData reform(long millisStart, ProcessorConfig config, Product product, boolean reservationUsage, String operationStr, String usageTypeStr, String description, double cost) {
+ protected ReformedMetaData reform(long millisStart, ProcessorConfig config, Product product, boolean reservationUsage, String operationStr, String usageTypeStr, String description, double cost, boolean credit) {
Operation operation = null;
UsageType usageType = null;
@@ -432,6 +492,9 @@ else if (usageTypeStr.startsWith("HeavyUsage") || usageTypeStr.startsWith("Mediu
usageType = UsageType.getUsageType(usageTypeStr, operation, description);
}
+ if (credit) {
+ product = new Product(product.name + " credit");
+ }
return new ReformedMetaData(region, product, operation, usageType);
}
From e390873d05a8bc08d12652a712a08b7157542979 Mon Sep 17 00:00:00 2001
From: Anthony Johnson <ansoni@gmail.com>
Date: Mon, 2 Mar 2015 15:31:45 -0500
Subject: [PATCH 2/8] -- re-work some of the credit handling -- update ice.js
to not ignore negative Y axis -- ignored records now have a debug statement.
---
.../ice/basic/BasicLineItemProcessor.java | 91 ++++----
web-app/js/ice.js | 216 ++++++++++++++++--
2 files changed, 250 insertions(+), 57 deletions(-)
diff --git a/src/java/com/netflix/ice/basic/BasicLineItemProcessor.java b/src/java/com/netflix/ice/basic/BasicLineItemProcessor.java
index 739cdf21..877a28f7 100644
--- a/src/java/com/netflix/ice/basic/BasicLineItemProcessor.java
+++ b/src/java/com/netflix/ice/basic/BasicLineItemProcessor.java
@@ -103,7 +103,11 @@ public Result process(long startMilli, boolean processDelayed, ProcessorConfig c
credit = true;
}
- //fail-fast on records we can't process
+ if (credit) {
+ if (! reformCredit(startMilli, items))
+ return Result.ignore;
+ }
+ // fail-fast on records we can't process
if (StringUtils.isEmpty(items[accountIdIndex])) {
logger.debug("Ignoring Record due to missing Account Id - " + Arrays.toString(items));
return Result.ignore;
@@ -112,49 +116,32 @@ public Result process(long startMilli, boolean processDelayed, ProcessorConfig c
return Result.ignore;
}
- // check other records. We might have to help some credit rows to show up properly
if (StringUtils.isEmpty(items[usageTypeIndex])) {
- if (credit) {
- items[usageTypeIndex]="credit";
- } else {
- logger.debug("Ignoring Record due to missing Usage Type - " + Arrays.toString(items));
- return Result.ignore;
- }
+ logger.debug("Ignoring Record due to missing Usage Type - " + Arrays.toString(items));
+ return Result.ignore;
}
if (StringUtils.isEmpty(items[operationIndex])) {
- if (credit) {
- items[operationIndex]="credit";
- } else {
- logger.debug("Ignoring Record due to missing Operation - " + Arrays.toString(items));
- return Result.ignore;
- }
+ logger.debug("Ignoring Record due to missing Operation - " + Arrays.toString(items));
+ return Result.ignore;
}
if (StringUtils.isEmpty(items[usageQuantityIndex])) {
- if (credit) {
- items[usageQuantityIndex]="1";
- } else {
- logger.debug("Ignoring Record due to missing Usage Quantity - " + Arrays.toString(items));
- return Result.ignore;
- }
+ items[usageQuantityIndex]="1";
+ logger.debug("Ignoring Record due to missing Usage Quantity - " + Arrays.toString(items));
+ return Result.ignore;
}
Account account = config.accountService.getAccountById(items[accountIdIndex]);
- if (account == null)
+ if (account == null) {
+ logger.debug("Ignoring Record due to missing Account - " + Arrays.toString(items));
return Result.ignore;
+ }
double usageValue = Double.parseDouble(items[usageQuantityIndex]);
long millisStart;
long millisEnd;
- boolean monthlyCredit=false;
Result result = Result.hourly;
- if (credit && items[startTimeIndex].isEmpty()) {
- items[startTimeIndex]=new DateTime(startMilli, DateTimeZone.UTC).toString(amazonBillingDateFormat);
- int numHoursInMonth = new DateTime(startMilli, DateTimeZone.UTC).dayOfMonth().getMaximumValue() * 24;
- items[endTimeIndex]=new DateTime(startMilli, DateTimeZone.UTC).plusHours(numHoursInMonth).toString(amazonBillingDateFormat);
- monthlyCredit=true;
- }
try {
millisStart = amazonBillingDateFormat.parseMillis(items[startTimeIndex]);
@@ -195,18 +182,15 @@ else if (product == Product.rds) {
result = processRds(usageType);
}
- // make sure credits for the month are distributed across month
-
- if (result == Result.ignore || result == Result.delay)
+ if (result == Result.ignore || result == Result.delay) {
+ logger.debug("Record not processed - " + result + " - " + Arrays.toString(items));
return result;
+ }
if (usageType.name.startsWith("TimedStorage-ByteHrs"))
result = Result.daily;
- if (monthlyCredit)
- result = Result.monthly;
-
- boolean monthlyCost = StringUtils.isEmpty(items[descriptionIndex]) ? false : ( items[descriptionIndex].toLowerCase().contains("-month") || monthlyCredit );
+ boolean monthlyCost = StringUtils.isEmpty(items[descriptionIndex]) ? false : ( items[descriptionIndex].toLowerCase().contains("-month") );
ReadWriteData usageData = usageDataByProduct.get(null);
ReadWriteData costData = costDataByProduct.get(null);
@@ -222,8 +206,6 @@ else if (product == Product.rds) {
else if (result == Result.monthly) {
startIndex = 0;
endIndex = usageData.getNum();
- logger.debug("Start Index: " + startIndex);
- logger.debug("End Index: " + endIndex);
int numHoursInMonth = new DateTime(startMilli, DateTimeZone.UTC).dayOfMonth().getMaximumValue() * 24;
usageValue = usageValue * endIndex / numHoursInMonth;
costValue = costValue * endIndex / numHoursInMonth;
@@ -291,10 +273,6 @@ else if (result == Result.monthly) {
return result;
for (int i : indexes) {
- if (credit) {
- logger.debug("Index: " + i);
- }
-
if (config.randomizer != null) {
if (tagGroup.product != Product.rds && tagGroup.product != Product.s3 && usageData.getData(i).get(tagGroup) != null)
@@ -405,6 +383,37 @@ private Result processRds(UsageType usageType) {
return Result.hourly;
}
+ protected boolean reformCredit(long startMilli, String[] items) {
+
+ String[] split_description = items[descriptionIndex].split(":");
+
+ // If the credit doesn't have a start and end time then we... set to end to end of month
+ if (items[startTimeIndex].isEmpty()) {
+ int numHoursInMonth = new DateTime(startMilli, DateTimeZone.UTC).dayOfMonth().getMaximumValue() * 24;
+ items[startTimeIndex]=new DateTime(startMilli, DateTimeZone.UTC).plusHours(numHoursInMonth-1).toString(amazonBillingDateFormat);
+ items[endTimeIndex]=new DateTime(startMilli, DateTimeZone.UTC).plusHours(numHoursInMonth).toString(amazonBillingDateFormat);
+ logger.debug("Credit did not have a Time - Set to " + items[startTimeIndex] + "-" + items[endTimeIndex]) ;
+ }
+
+ // try to use the description to fetch some info about the credit if we have nothing
+ if (items[operationIndex].isEmpty()) {
+ if (split_description.length > 0)
+ items[operationIndex]=split_description[0];
+ else
+ items[operationIndex]="credit";
+ }
+
+ if (items[usageTypeIndex].isEmpty()) {
+ items[usageTypeIndex]="credit";
+ }
+
+ if (items[usageQuantityIndex].isEmpty())
+ items[usageQuantityIndex]="1";
+
+ return true;
+
+ }
+
protected ReformedMetaData reform(long millisStart, ProcessorConfig config, Product product, boolean reservationUsage, String operationStr, String usageTypeStr, String description, double cost, boolean credit) {
Operation operation = null;
diff --git a/web-app/js/ice.js b/web-app/js/ice.js
index f76e6011..22df2cdc 100644
--- a/web-app/js/ice.js
+++ b/web-app/js/ice.js
@@ -59,6 +59,7 @@ ice.factory('highchart', function() {
plotOptions: {
area: {lineWidth: 1, stacking: 'normal'},
column: {lineWidth: 1, stacking: 'normal'},
+ line: {lineWidth: 1, stacking: null},
series: {
states: {
hover: {
@@ -92,11 +93,11 @@ ice.factory('highchart', function() {
var precision = currencySign === "" ? 0 : (currencySign === "¢" ? 4 : 2);
for (var i = 0; i < this.points.length - (showsps ? 1 : 0); i++) {
var point = this.points[i];
- if (i == 0) {
+ if (i == 0 && point.total!=undefined) {
s += '<br/><span>aggregated : ' + currencySign + Highcharts.numberFormat(showsps ? total : point.total, precision, '.') + ' / ' + (factorsps ? metricunitname : consolidate);
}
var perc = showsps ? point.y * 100 / total : point.percentage;
- s += '<br/><span style="color: ' + point.series.color + '">' + point.series.name + '</span> : ' + currencySign + Highcharts.numberFormat(point.y, precision, '.') + ' / ' + (factorsps ? metricunitname : consolidate) + ' (' + Highcharts.numberFormat(perc, 1) + '%)';
+ s += '<br/><span style="color: ' + point.series.color + '">' + point.series.name + '</span> : ' + currencySign + Highcharts.numberFormat(point.y, precision, '.') + ' / ' + (factorsps ? metricunitname : consolidate) + ' (' + Highcharts.numberFormat(perc, 1) + '%)';
if (i > 40 && point)
break;
}
@@ -106,7 +107,55 @@ ice.factory('highchart', function() {
}
};
- var setupHcData = function(result, plotType, showsps) {
+ var setupHcEstimate = function(result, plotType, cumulative) {
+ plotType = "line";
+ Highcharts.setOptions({
+ global: {
+ useUTC: true
+ }
+ });
+
+ var hadEstimate = false;
+
+ for (i in result.estimates) {
+ var estimates = result.estimates[i].data;
+ var hasEstimate = false;
+ for (j in estimates) {
+ if (cumulative && cumulative == "true" && j > 0) {
+ estimates[j] = estimates[j-1] + parseFloat(estimates[j].toFixed(2));
+ //aggregateEstimates[j] += estimates[j];
+ } else {
+ estimates[j] = parseFloat(estimates[j].toFixed(2));
+ //aggregateEstimates[j] += estimates[j];
+ }
+ if (estimates[j] !== 0)
+ hasEstimate = true;
+ }
+
+ if (hasEstimate) {
+ hadEstimate = true;
+ if (!result.interval && result.time) {
+ for (j in estimates) {
+ estimates[j] = [result.time[j], estimates[j]];
+ }
+ }
+
+ var serie = {
+ name: result.estimates[i].name + " Estimate",
+ data: estimates,
+ pointStart: result.start,
+ pointInterval: result.interval,
+ //step: true,
+ type: "line",
+ Index: -1
+ };
+
+ hc_options.series.push(serie);
+ }
+ }
+ }
+
+ var setupHcData = function(result, plotType, showsps, cumulative) {
Highcharts.setOptions({
global: {
@@ -120,7 +169,10 @@ ice.factory('highchart', function() {
var data = result.data[i].data;
var hasData = false;
for (j in data) {
- data[j] = parseFloat(data[j].toFixed(2));
+ if (cumulative && cumulative == "true" && j > 0)
+ data[j] = data[j-1] + parseFloat(data[j].toFixed(2));
+ else
+ data[j] = parseFloat(data[j].toFixed(2));
if (data[j] !== 0)
hasData = true;
}
@@ -159,7 +211,7 @@ ice.factory('highchart', function() {
}
var setupYAxis = function(isCost, showsps, factorsps) {
- var yAxis = {title:{text: (isCost ? 'Cost' : 'Usage') + " per " + (factorsps ? metricunitname : consolidate)}, min: 0, lineWidth: 2};
+ var yAxis = {title:{text: (isCost ? 'Cost' : 'Usage') + " per " + (factorsps ? metricunitname : consolidate)}, lineWidth: 2};
if (isCost)
yAxis.labels = {
formatter: function() {
@@ -169,7 +221,7 @@ ice.factory('highchart', function() {
hc_options.yAxis = [yAxis];
if (showsps) {
- hc_options.yAxis.push({title:{text:metricname}, height: 100, min: 0, lineWidth: 2, offset: 0});
+ hc_options.yAxis.push({title:{text:metricname}, height: 100, lineWidth: 2, offset: 0});
hc_options.yAxis[0].top = 150;
hc_options.yAxis[0].height = 350;
}
@@ -195,7 +247,8 @@ ice.factory('highchart', function() {
currencySign = $scope.usage_cost === 'cost' ? ($scope.factorsps ? factoredCostCurrencySign : global_currencySign) : "";
hc_options.legend.enabled = legendEnabled;
- setupHcData(result, $scope.plotType, $scope.showsps);
+ setupHcData(result, $scope.plotType, $scope.showsps, $scope.cumulative);
+ setupHcEstimate(result, $scope.plotType, $scope.cumulative);
setupYAxis($scope.usage_cost === 'cost', $scope.showsps, $scope.factorsps);
showsps = $scope.showsps;
factorsps = $scope.factorsps;
@@ -210,7 +263,7 @@ ice.factory('highchart', function() {
var i = 0;
for (i = 0; i < chart.series.length - ($scope.showsps ? 2 : 1); i++) {
if ($scope && $scope.legends) {
- var legend = {
+ var legend = {
name: chart.series[i].name,
style: "color: " + chart.series[i].color,
iconStyle: "background-color: " + chart.series[i].color,
@@ -365,7 +418,7 @@ ice.factory('usage_db', function($window, $http, $filter) {
}
$location.hash(result);
-
+
if (time) {
timeParams = time;
}
@@ -380,9 +433,9 @@ ice.factory('usage_db', function($window, $http, $filter) {
if (hash) {
var params = hash.split("&");
for (i = 0; i < params.length; i++) {
- if (params[i].indexOf("=") < 0 && i > 0 && (params[i-1].indexOf("appgroup=") == 0 || params[i-1].indexOf("resourceGroup=") == 0))
- params[i-1] = params[i-1] + "&" + params[i];
- }
+ if (params[i].indexOf("=") < 0 && i > 0 && (params[i-1].indexOf("appgroup=") == 0 || params[i-1].indexOf("resourceGroup=") == 0))
+ params[i-1] = params[i-1] + "&" + params[i];
+ }
var i, j, time = "";
for (i = 0; i < params.length; i++) {
@@ -458,6 +511,9 @@ ice.factory('usage_db', function($window, $http, $filter) {
else if (params[i].indexOf("resourceGroup=") === 0) {
$scope.selected__resourceGroups = params[i].substr(14).split(",");
}
+ else if (params[i].indexOf("includeEstimates=") === 0) {
+ $scope.includeEstimates = "true" === params[i].substr(17);
+ }
}
}
if (!$scope.showResourceGroups) {
@@ -687,7 +743,7 @@ ice.factory('usage_db', function($window, $http, $filter) {
showsps: $scope.showsps ? true : false,
factorsps: $scope.factorsps ? true : false
}, params);
- this.addParams(params, "account", $scope.accounts, $scope.selected_accounts, $scope.selected__accounts, $scope.filter_accounts);
+ this.addParams(params, "account", $scope.accounts, $scope.selected__accounts, $scope.filter_accounts);
if ($scope.showZones)
this.addParams(params, "zone", $scope.zones, $scope.selected_zones, $scope.selected__zones, $scope.filter_zones);
else
@@ -1170,6 +1226,135 @@ function detailCtrl($scope, $location, $http, usage_db, highchart) {
fn();
}
+function estimateCtrl($scope, $location, $http, usage_db, highchart) {
+
+ $scope.graphType = "aggregate";
+ $scope.legends = [];
+ $scope.usage_cost = "cost";
+ $scope.includeEstimates=true;
+ $scope.cumulative="true";
+ $scope.dailyEstimate=500;
+ $scope.groupBy={ name: "Account" }
+ $scope.consolidate = "daily";
+ $scope.end = new Date();
+ $scope.start = new Date();
+ $scope.plotType = "area";
+
+ var startMonth = $scope.end.getUTCMonth() - 1;
+ var startYear = $scope.end.getUTCFullYear();
+ if (startMonth < 0) {
+ startMonth += 12;
+ startYear -= 1;
+ }
+ $scope.start.setUTCFullYear(startYear);
+ $scope.start.setUTCMonth(startMonth);
+ $scope.start.setUTCDate(1);
+ $scope.start.setUTCHours(0);
+
+ $scope.end = highchart.dateFormat($scope.end); //$filter('date')($scope.end, "y-MM-dd hha");
+ $scope.start = highchart.dateFormat($scope.start); //$filter('date')($scope.start, "y-MM-dd hha");
+
+
+ $scope.updateUrl = function() {
+ $scope.end = jQuery('#end').datetimepicker().val();
+ $scope.start = jQuery('#start').datetimepicker().val();
+
+ var params = {
+ account: {selected: $scope.selected_accounts, from: $scope.accounts},
+ usage_cost: $scope.usage_cost,
+ start: $scope.start,
+ end: $scope.end,
+ groupBy: "Account",
+ plotType: "area",
+ aggregate: "none",
+ consolidate: $scope.consolidate,
+ graphType: $scope.graphType,
+ includeEstimates: "" + $scope.includeEstimates
+ };
+ usage_db.updateUrl($location, params);
+ }
+
+ $scope.download = function() {
+ usage_db.getData($scope, null, null, true);
+ }
+
+ $scope.getData = function() {
+ $scope.loading = true;
+ usage_db.getData($scope, function(result){
+ var hourlydata = [];
+ var estimatedata = [];
+ for (var key in result.data) {
+ hourlydata.push({name: key, data: result.data[key]});
+ }
+ for (var key in result.estimates) {
+ estimatedata.push({name: key, data: result.estimates[key]});
+ }
+ result.data = hourlydata;
+ result.estimates = estimatedata;
+ $scope.legends = [];
+ $scope.stats = result.stats;
+ highchart.drawGraph(result, $scope);
+
+ $scope.legendName = $scope.groupBy.name;
+ $scope.legend_usage_cost = $scope.usage_cost;
+ }, { includeEstimates: "" + $scope.includeEstimates});
+ }
+
+ $scope.accountsChanged = function() {
+ $scope.updateEstimates();
+ }
+
+ $scope.updateEstimates = function() {
+ usage_db.getDailyEstimate($scope);
+ }
+
+ $scope.updateUsageTypes = function() {
+ usage_db.getUsageTypes($scope, function(data){
+ });
+ }
+
+ $scope.removeEstimatesFilter = function(legend) {
+ return legend.name.indexOf("Estimate") == -1;
+ };
+
+
+ var fn = function() {
+ usage_db.getAccounts($scope, function(data){
+ //while (! $scope.selected_account) {
+ //we need to have these!
+ // }
+ });
+ $scope.getData();
+
+ jQuery("#start, #end" ).datetimepicker({
+ showTime: false,
+ showMinute: false,
+ ampm: true,
+ timeFormat: 'hhTT',
+ dateFormat: 'yy-mm-dd'
+ });
+ jQuery('#end').datetimepicker().val($scope.end);
+ jQuery('#start').datetimepicker().val($scope.start);
+ }
+
+ usage_db.getParams($location.hash(), $scope);
+
+ if ($scope.spans) {
+ $http({
+ method: "GET",
+ url: "getTimeSpan",
+ params: {spans: $scope.spans, end: $scope.end, consolidate: $scope.consolidate}
+ }).success(function(result) {
+ $scope.end = result.end;
+ $scope.start = result.start;
+ fn();
+ });
+ } else {
+ fn();
+ }
+}
+
+
function appgroupCtrl($scope, $location, $http, usage_db, highchart) {
// var predefinedQuery = {product: "ebs,ec2,ec2_instance,monitor,rds,s3"};
@@ -1496,7 +1681,7 @@ function summaryCtrl($scope, $location, usage_db, highchart) {
{name: "UsageType"}
],
$scope.groupBy = $scope.groupBys[2];
- $scope.consolidate = "hourly";
+ $scope.consolidate = "monthly";
$scope.plotType = "area";
$scope.end = new Date();
$scope.start = new Date();
@@ -1526,7 +1711,7 @@ function summaryCtrl($scope, $location, usage_db, highchart) {
$scope.order = function(index) {
if ($scope.predicate != index) {
- $scope.reservse = index === 'name';
+ $scope.reserve = index === 'name';
$scope.predicate = index;
}
else {
@@ -1804,4 +1989,3 @@ function editCtrl($scope, $location, $http) {
}
});
}
-
From 33386d86832d3bc8394ac52bbaf744697a582774 Mon Sep 17 00:00:00 2001
From: Anthony Johnson <ansoni@gmail.com>
Date: Mon, 2 Mar 2015 15:51:12 -0500
Subject: [PATCH 3/8] Revert accidental modifications pulled in from another
feature branch
---
web-app/js/ice.js | 212 +++-------------------------------------------
1 file changed, 14 insertions(+), 198 deletions(-)
diff --git a/web-app/js/ice.js b/web-app/js/ice.js
index 22df2cdc..9daaac01 100644
--- a/web-app/js/ice.js
+++ b/web-app/js/ice.js
@@ -59,7 +59,6 @@ ice.factory('highchart', function() {
plotOptions: {
area: {lineWidth: 1, stacking: 'normal'},
column: {lineWidth: 1, stacking: 'normal'},
- line: {lineWidth: 1, stacking: null},
series: {
states: {
hover: {
@@ -93,11 +92,11 @@ ice.factory('highchart', function() {
var precision = currencySign === "" ? 0 : (currencySign === "¢" ? 4 : 2);
for (var i = 0; i < this.points.length - (showsps ? 1 : 0); i++) {
var point = this.points[i];
- if (i == 0 && point.total!=undefined) {
+ if (i == 0) {
s += '<br/><span>aggregated : ' + currencySign + Highcharts.numberFormat(showsps ? total : point.total, precision, '.') + ' / ' + (factorsps ? metricunitname : consolidate);
}
var perc = showsps ? point.y * 100 / total : point.percentage;
- s += '<br/><span style="color: ' + point.series.color + '">' + point.series.name + '</span> : ' + currencySign + Highcharts.numberFormat(point.y, precision, '.') + ' / ' + (factorsps ? metricunitname : consolidate) + ' (' + Highcharts.numberFormat(perc, 1) + '%)';
+ s += '<br/><span style="color: ' + point.series.color + '">' + point.series.name + '</span> : ' + currencySign + Highcharts.numberFormat(point.y, precision, '.') + ' / ' + (factorsps ? metricunitname : consolidate) + ' (' + Highcharts.numberFormat(perc, 1) + '%)';
if (i > 40 && point)
break;
}
@@ -107,55 +106,7 @@ ice.factory('highchart', function() {
}
};
- var setupHcEstimate = function(result, plotType, cumulative) {
- plotType = "line";
- Highcharts.setOptions({
- global: {
- useUTC: true
- }
- });
-
- var hadEstimate = false;
-
- for (i in result.estimates) {
- var estimates = result.estimates[i].data;
- var hasEstimate = false;
- for (j in estimates) {
- if (cumulative && cumulative == "true" && j > 0) {
- estimates[j] = estimates[j-1] + parseFloat(estimates[j].toFixed(2));
- //aggregateEstimates[j] += estimates[j];
- } else {
- estimates[j] = parseFloat(estimates[j].toFixed(2));
- //aggregateEstimates[j] += estimates[j];
- }
- if (estimates[j] !== 0)
- hasEstimate = true;
- }
-
- if (hasEstimate) {
- hadEstimate = true;
- if (!result.interval && result.time) {
- for (j in estimates) {
- estimates[j] = [result.time[j], estimates[j]];
- }
- }
-
- var serie = {
- name: result.estimates[i].name + " Estimate",
- data: estimates,
- pointStart: result.start,
- pointInterval: result.interval,
- //step: true,
- type: "line",
- Index: -1
- };
-
- hc_options.series.push(serie);
- }
- }
- }
-
- var setupHcData = function(result, plotType, showsps, cumulative) {
+ var setupHcData = function(result, plotType, showsps) {
Highcharts.setOptions({
global: {
@@ -169,10 +120,7 @@ ice.factory('highchart', function() {
var data = result.data[i].data;
var hasData = false;
for (j in data) {
- if (cumulative && cumulative == "true" && j > 0)
- data[j] = data[j-1] + parseFloat(data[j].toFixed(2));
- else
- data[j] = parseFloat(data[j].toFixed(2));
+ data[j] = parseFloat(data[j].toFixed(2));
if (data[j] !== 0)
hasData = true;
}
@@ -247,8 +195,7 @@ ice.factory('highchart', function() {
currencySign = $scope.usage_cost === 'cost' ? ($scope.factorsps ? factoredCostCurrencySign : global_currencySign) : "";
hc_options.legend.enabled = legendEnabled;
- setupHcData(result, $scope.plotType, $scope.showsps, $scope.cumulative);
- setupHcEstimate(result, $scope.plotType, $scope.cumulative);
+ setupHcData(result, $scope.plotType, $scope.showsps);
setupYAxis($scope.usage_cost === 'cost', $scope.showsps, $scope.factorsps);
showsps = $scope.showsps;
factorsps = $scope.factorsps;
@@ -263,7 +210,7 @@ ice.factory('highchart', function() {
var i = 0;
for (i = 0; i < chart.series.length - ($scope.showsps ? 2 : 1); i++) {
if ($scope && $scope.legends) {
- var legend = {
+ var legend = {
name: chart.series[i].name,
style: "color: " + chart.series[i].color,
iconStyle: "background-color: " + chart.series[i].color,
@@ -418,7 +365,7 @@ ice.factory('usage_db', function($window, $http, $filter) {
}
$location.hash(result);
-
+
if (time) {
timeParams = time;
}
@@ -433,9 +380,9 @@ ice.factory('usage_db', function($window, $http, $filter) {
if (hash) {
var params = hash.split("&");
for (i = 0; i < params.length; i++) {
- if (params[i].indexOf("=") < 0 && i > 0 && (params[i-1].indexOf("appgroup=") == 0 || params[i-1].indexOf("resourceGroup=") == 0))
- params[i-1] = params[i-1] + "&" + params[i];
- }
+ if (params[i].indexOf("=") < 0 && i > 0 && (params[i-1].indexOf("appgroup=") == 0 || params[i-1].indexOf("resourceGroup=") == 0))
+ params[i-1] = params[i-1] + "&" + params[i];
+ }
var i, j, time = "";
for (i = 0; i < params.length; i++) {
@@ -511,9 +458,6 @@ ice.factory('usage_db', function($window, $http, $filter) {
else if (params[i].indexOf("resourceGroup=") === 0) {
$scope.selected__resourceGroups = params[i].substr(14).split(",");
}
- else if (params[i].indexOf("includeEstimates=") === 0) {
- $scope.includeEstimates = "true" === params[i].substr(17);
- }
}
}
if (!$scope.showResourceGroups) {
@@ -743,7 +687,7 @@ ice.factory('usage_db', function($window, $http, $filter) {
showsps: $scope.showsps ? true : false,
factorsps: $scope.factorsps ? true : false
}, params);
- this.addParams(params, "account", $scope.accounts, $scope.selected__accounts, $scope.filter_accounts);
+ this.addParams(params, "account", $scope.accounts, $scope.selected_accounts, $scope.selected__accounts, $scope.filter_accounts);
if ($scope.showZones)
this.addParams(params, "zone", $scope.zones, $scope.selected_zones, $scope.selected__zones, $scope.filter_zones);
else
@@ -1226,135 +1170,6 @@ function detailCtrl($scope, $location, $http, usage_db, highchart) {
fn();
}
-function estimateCtrl($scope, $location, $http, usage_db, highchart) {
-
- $scope.graphType = "aggregate";
- $scope.legends = [];
- $scope.usage_cost = "cost";
- $scope.includeEstimates=true;
- $scope.cumulative="true";
- $scope.dailyEstimate=500;
- $scope.groupBy={ name: "Account" }
- $scope.consolidate = "daily";
- $scope.end = new Date();
- $scope.start = new Date();
- $scope.plotType = "area";
-
- var startMonth = $scope.end.getUTCMonth() - 1;
- var startYear = $scope.end.getUTCFullYear();
- if (startMonth < 0) {
- startMonth += 12;
- startYear -= 1;
- }
- $scope.start.setUTCFullYear(startYear);
- $scope.start.setUTCMonth(startMonth);
- $scope.start.setUTCDate(1);
- $scope.start.setUTCHours(0);
-
- $scope.end = highchart.dateFormat($scope.end); //$filter('date')($scope.end, "y-MM-dd hha");
- $scope.start = highchart.dateFormat($scope.start); //$filter('date')($scope.start, "y-MM-dd hha");
-
-
- $scope.updateUrl = function() {
- $scope.end = jQuery('#end').datetimepicker().val();
- $scope.start = jQuery('#start').datetimepicker().val();
-
- var params = {
- account: {selected: $scope.selected_accounts, from: $scope.accounts},
- usage_cost: $scope.usage_cost,
- start: $scope.start,
- end: $scope.end,
- groupBy: "Account",
- plotType: "area",
- aggregate: "none",
- consolidate: $scope.consolidate,
- graphType: $scope.graphType,
- includeEstimates: "" + $scope.includeEstimates
- };
- usage_db.updateUrl($location, params);
- }
-
- $scope.download = function() {
- usage_db.getData($scope, null, null, true);
- }
-
- $scope.getData = function() {
- $scope.loading = true;
- usage_db.getData($scope, function(result){
- var hourlydata = [];
- var estimatedata = [];
- for (var key in result.data) {
- hourlydata.push({name: key, data: result.data[key]});
- }
- for (var key in result.estimates) {
- estimatedata.push({name: key, data: result.estimates[key]});
- }
- result.data = hourlydata;
- result.estimates = estimatedata;
- $scope.legends = [];
- $scope.stats = result.stats;
- highchart.drawGraph(result, $scope);
-
- $scope.legendName = $scope.groupBy.name;
- $scope.legend_usage_cost = $scope.usage_cost;
- }, { includeEstimates: "" + $scope.includeEstimates});
- }
-
- $scope.accountsChanged = function() {
- $scope.updateEstimates();
- }
-
- $scope.updateEstimates = function() {
- usage_db.getDailyEstimate($scope);
- }
-
- $scope.updateUsageTypes = function() {
- usage_db.getUsageTypes($scope, function(data){
- });
- }
-
- $scope.removeEstimatesFilter = function(legend) {
- return legend.name.indexOf("Estimate") == -1;
- };
-
-
- var fn = function() {
- usage_db.getAccounts($scope, function(data){
- //while (! $scope.selected_account) {
- //we need to have these!
- // }
- });
- $scope.getData();
-
- jQuery("#start, #end" ).datetimepicker({
- showTime: false,
- showMinute: false,
- ampm: true,
- timeFormat: 'hhTT',
- dateFormat: 'yy-mm-dd'
- });
- jQuery('#end').datetimepicker().val($scope.end);
- jQuery('#start').datetimepicker().val($scope.start);
- }
-
- usage_db.getParams($location.hash(), $scope);
-
- if ($scope.spans) {
- $http({
- method: "GET",
- url: "getTimeSpan",
- params: {spans: $scope.spans, end: $scope.end, consolidate: $scope.consolidate}
- }).success(function(result) {
- $scope.end = result.end;
- $scope.start = result.start;
- fn();
- });
- } else {
- fn();
- }
-}
-
-
function appgroupCtrl($scope, $location, $http, usage_db, highchart) {
// var predefinedQuery = {product: "ebs,ec2,ec2_instance,monitor,rds,s3"};
@@ -1681,7 +1496,7 @@ function summaryCtrl($scope, $location, usage_db, highchart) {
{name: "UsageType"}
],
$scope.groupBy = $scope.groupBys[2];
- $scope.consolidate = "monthly";
+ $scope.consolidate = "hourly";
$scope.plotType = "area";
$scope.end = new Date();
$scope.start = new Date();
@@ -1711,7 +1526,7 @@ function summaryCtrl($scope, $location, usage_db, highchart) {
$scope.order = function(index) {
if ($scope.predicate != index) {
- $scope.reserve = index === 'name';
+ $scope.reservse = index === 'name';
$scope.predicate = index;
}
else {
@@ -1989,3 +1804,4 @@ function editCtrl($scope, $location, $http) {
}
});
}
+
From 30e466c029f44ce98613b1477ceb84d6b9484b95 Mon Sep 17 00:00:00 2001
From: Anthony Johnson <ansoni@gmail.com>
Date: Mon, 2 Mar 2015 15:57:52 -0500
Subject: [PATCH 4/8] small cleanup
---
src/java/com/netflix/ice/basic/BasicLineItemProcessor.java | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/src/java/com/netflix/ice/basic/BasicLineItemProcessor.java b/src/java/com/netflix/ice/basic/BasicLineItemProcessor.java
index 877a28f7..c9091967 100644
--- a/src/java/com/netflix/ice/basic/BasicLineItemProcessor.java
+++ b/src/java/com/netflix/ice/basic/BasicLineItemProcessor.java
@@ -111,11 +111,11 @@ public Result process(long startMilli, boolean processDelayed, ProcessorConfig c
if (StringUtils.isEmpty(items[accountIdIndex])) {
logger.debug("Ignoring Record due to missing Account Id - " + Arrays.toString(items));
return Result.ignore;
- } else if (StringUtils.isEmpty(items[productIndex])) {
+ }
+ if (StringUtils.isEmpty(items[productIndex])) {
logger.debug("Ignoring Record due to missing Product - " + Arrays.toString(items));
return Result.ignore;
}
-
if (StringUtils.isEmpty(items[usageTypeIndex])) {
logger.debug("Ignoring Record due to missing Usage Type - " + Arrays.toString(items));
return Result.ignore;
@@ -125,7 +125,6 @@ public Result process(long startMilli, boolean processDelayed, ProcessorConfig c
return Result.ignore;
}
if (StringUtils.isEmpty(items[usageQuantityIndex])) {
- items[usageQuantityIndex]="1";
logger.debug("Ignoring Record due to missing Usage Quantity - " + Arrays.toString(items));
return Result.ignore;
}
From 89aba5e971c88f322db266ab82906de53f400ea9 Mon Sep 17 00:00:00 2001
From: Anthony Johnson <ansoni@gmail.com>
Date: Thu, 9 Apr 2015 15:19:58 -0400
Subject: [PATCH 5/8] Updates to credit processing to ensure we capture all
credit entries properly.
---
.../ice/basic/BasicLineItemProcessor.java | 34 ++++++++++++-------
1 file changed, 21 insertions(+), 13 deletions(-)
diff --git a/src/java/com/netflix/ice/basic/BasicLineItemProcessor.java b/src/java/com/netflix/ice/basic/BasicLineItemProcessor.java
index c9091967..98a425a6 100644
--- a/src/java/com/netflix/ice/basic/BasicLineItemProcessor.java
+++ b/src/java/com/netflix/ice/basic/BasicLineItemProcessor.java
@@ -101,12 +101,12 @@ public Result process(long startMilli, boolean processDelayed, ProcessorConfig c
// make sure we don't ignore credits
if (costValue < 0) {
credit = true;
- }
-
- if (credit) {
- if (! reformCredit(startMilli, items))
+ if (! reformCredit(startMilli, items)) {
return Result.ignore;
+ }
+ logger.debug("Found Credit - " + Arrays.toString(items));
}
+
// fail-fast on records we can't process
if (StringUtils.isEmpty(items[accountIdIndex])) {
logger.debug("Ignoring Record due to missing Account Id - " + Arrays.toString(items));
@@ -251,7 +251,7 @@ else if (result == Result.monthly) {
}
catch (Exception e) {
logger.error("failed to get RI price for " + tagGroup.region + " " + usageTypeForPrice);
- resourceCostValue = -1;
+ resourceCostValue = Double.MIN_VALUE;
}
}
@@ -272,6 +272,9 @@ else if (result == Result.monthly) {
return result;
for (int i : indexes) {
+ if (credit) {
+ logger.debug("Handled Credit Index " + i + " - " + costValue);
+ }
if (config.randomizer != null) {
if (tagGroup.product != Product.rds && tagGroup.product != Product.s3 && usageData.getData(i).get(tagGroup) != null)
@@ -285,8 +288,8 @@ else if (result == Result.monthly) {
Map<TagGroup, Double> usages = usageData.getData(i);
Map<TagGroup, Double> costs = costData.getData(i);
- addValue(usages, tagGroup, usageValue, config.randomizer == null || tagGroup.product == Product.rds || tagGroup.product == Product.s3);
- addValue(costs, tagGroup, costValue, config.randomizer == null || tagGroup.product == Product.rds || tagGroup.product == Product.s3);
+ addValue(usages, tagGroup, usageValue, config.randomizer == null || tagGroup.product == Product.rds || tagGroup.product == Product.s3 || credit == true);
+ addValue(costs, tagGroup, costValue, config.randomizer == null || tagGroup.product == Product.rds || tagGroup.product == Product.s3 || credit == true);
}
else {
resourceCostValue = usageValue * config.costPerMonitorMetricPerHour;
@@ -296,9 +299,9 @@ else if (result == Result.monthly) {
Map<TagGroup, Double> usagesOfResource = usageDataOfProduct.getData(i);
Map<TagGroup, Double> costsOfResource = costDataOfProduct.getData(i);
- if (config.randomizer == null || tagGroup.product == Product.rds || tagGroup.product == Product.s3) {
+ if (config.randomizer == null || tagGroup.product == Product.rds || tagGroup.product == Product.s3 || credit == true) {
addValue(usagesOfResource, resourceTagGroup, usageValue, product != Product.monitor);
- if (!config.useCostForResourceGroup.equals("modeled") || resourceCostValue < 0) {
+ if (!config.useCostForResourceGroup.equals("modeled") || resourceCostValue == Double.MIN_VALUE) {
addValue(costsOfResource, resourceTagGroup, costValue, product != Product.monitor);
} else {
addValue(costsOfResource, resourceTagGroup, resourceCostValue, product != Product.monitor);
@@ -394,6 +397,9 @@ protected boolean reformCredit(long startMilli, String[] items) {
logger.debug("Credit did not have a Time - Set to " + items[startTimeIndex] + "-" + items[endTimeIndex]) ;
}
+ // seperate credits into their own product for easy filtering/aggregation
+ items[productIndex]+=" credit";
+
// try to use the description to fetch some info about the credit if we have nothing
if (items[operationIndex].isEmpty()) {
if (split_description.length > 0)
@@ -406,8 +412,9 @@ protected boolean reformCredit(long startMilli, String[] items) {
items[usageTypeIndex]="credit";
}
- if (items[usageQuantityIndex].isEmpty())
- items[usageQuantityIndex]="1";
+ if (items[usageQuantityIndex].isEmpty()) {
+ items[usageQuantityIndex] = "1";
+ }
return true;
@@ -500,9 +507,10 @@ else if (usageTypeStr.startsWith("HeavyUsage") || usageTypeStr.startsWith("Mediu
usageType = UsageType.getUsageType(usageTypeStr, operation, description);
}
- if (credit) {
+ // This method resets product. Make sure we seperated out our credits
+ if (credit && ! product.name.endsWith("credit"))
product = new Product(product.name + " credit");
- }
+
return new ReformedMetaData(region, product, operation, usageType);
}
From 3c315c0b7269ef52f7638e3f46910fdb849067be Mon Sep 17 00:00:00 2001
From: Anthony Johnson <ansoni@gmail.com>
Date: Mon, 13 Jul 2015 14:30:03 -0400
Subject: [PATCH 6/8] Add Support to turn Credit Support On/Off
---
src/java/com/netflix/ice/basic/BasicLineItemProcessor.java | 2 +-
src/java/com/netflix/ice/common/IceOptions.java | 5 +++++
src/java/com/netflix/ice/processor/ProcessorConfig.java | 2 ++
3 files changed, 8 insertions(+), 1 deletion(-)
diff --git a/src/java/com/netflix/ice/basic/BasicLineItemProcessor.java b/src/java/com/netflix/ice/basic/BasicLineItemProcessor.java
index 98a425a6..0392071b 100644
--- a/src/java/com/netflix/ice/basic/BasicLineItemProcessor.java
+++ b/src/java/com/netflix/ice/basic/BasicLineItemProcessor.java
@@ -101,7 +101,7 @@ public Result process(long startMilli, boolean processDelayed, ProcessorConfig c
// make sure we don't ignore credits
if (costValue < 0) {
credit = true;
- if (! reformCredit(startMilli, items)) {
+ if (config.ignoreCredits || ! reformCredit(startMilli, items)) {
return Result.ignore;
}
logger.debug("Found Credit - " + Arrays.toString(items));
diff --git a/src/java/com/netflix/ice/common/IceOptions.java b/src/java/com/netflix/ice/common/IceOptions.java
index c7d9bca9..0bf10f92 100644
--- a/src/java/com/netflix/ice/common/IceOptions.java
+++ b/src/java/com/netflix/ice/common/IceOptions.java
@@ -100,6 +100,11 @@ public class IceOptions {
*/
public static final String MONTHLY_CACHE_SIZE = "ice.monthlycachesize";
+ /**
+ * Should we ignore credits or not?
+ */
+ public static final String IGNORE_CREDITS = "ice.ignore_credits";
+
/**
* Cost per monitor metric per hour, It's optional.
*/
diff --git a/src/java/com/netflix/ice/processor/ProcessorConfig.java b/src/java/com/netflix/ice/processor/ProcessorConfig.java
index c009dc4e..1fda642b 100644
--- a/src/java/com/netflix/ice/processor/ProcessorConfig.java
+++ b/src/java/com/netflix/ice/processor/ProcessorConfig.java
@@ -41,6 +41,7 @@ public class ProcessorConfig extends Config {
public final LineItemProcessor lineItemProcessor;
public final Randomizer randomizer;
public final double costPerMonitorMetricPerHour;
+ public final boolean ignoreCredits;
public final String useCostForResourceGroup;
@@ -87,6 +88,7 @@ public ProcessorConfig(
customTags = properties.getProperty(IceOptions.CUSTOM_TAGS, "").split(",");
useCostForResourceGroup = properties.getProperty(IceOptions.RESOURCE_GROUP_COST, "modeled");
+ ignoreCredits = Boolean.parseBoolean(properties.getProperty(IceOptions.IGNORE_CREDITS, "true"));
ProcessorConfig.instance = this;
From 6283e941da3554e8539109b259288ebdec195b8b Mon Sep 17 00:00:00 2001
From: Anthony Johnson <ansoni@gmail.com>
Date: Mon, 13 Jul 2015 17:39:30 -0400
Subject: [PATCH 7/8] Move debug to info for ignored and credit line-items
---
.../ice/basic/BasicLineItemProcessor.java | 18 +++++++++---------
1 file changed, 9 insertions(+), 9 deletions(-)
diff --git a/src/java/com/netflix/ice/basic/BasicLineItemProcessor.java b/src/java/com/netflix/ice/basic/BasicLineItemProcessor.java
index 0392071b..d1a0cede 100644
--- a/src/java/com/netflix/ice/basic/BasicLineItemProcessor.java
+++ b/src/java/com/netflix/ice/basic/BasicLineItemProcessor.java
@@ -91,7 +91,7 @@ public long getEndMillis(String[] items) {
public Result process(long startMilli, boolean processDelayed, ProcessorConfig config, String[] items, Map<Product, ReadWriteData> usageDataByProduct, Map<Product, ReadWriteData> costDataByProduct, Map<String, Double> ondemandRate) {
if (StringUtils.isEmpty(items[costIndex])) {
- logger.debug("Ignoring Record due to missing Cost - " + Arrays.toString(items));
+ logger.info("Ignoring Record due to missing Cost - " + Arrays.toString(items));
return Result.ignore;
}
@@ -104,34 +104,34 @@ public Result process(long startMilli, boolean processDelayed, ProcessorConfig c
if (config.ignoreCredits || ! reformCredit(startMilli, items)) {
return Result.ignore;
}
- logger.debug("Found Credit - " + Arrays.toString(items));
+ logger.info("Found Credit - " + Arrays.toString(items));
}
// fail-fast on records we can't process
if (StringUtils.isEmpty(items[accountIdIndex])) {
- logger.debug("Ignoring Record due to missing Account Id - " + Arrays.toString(items));
+ logger.info("Ignoring Record due to missing Account Id - " + Arrays.toString(items));
return Result.ignore;
}
if (StringUtils.isEmpty(items[productIndex])) {
- logger.debug("Ignoring Record due to missing Product - " + Arrays.toString(items));
+ logger.info("Ignoring Record due to missing Product - " + Arrays.toString(items));
return Result.ignore;
}
if (StringUtils.isEmpty(items[usageTypeIndex])) {
- logger.debug("Ignoring Record due to missing Usage Type - " + Arrays.toString(items));
+ logger.info("Ignoring Record due to missing Usage Type - " + Arrays.toString(items));
return Result.ignore;
}
if (StringUtils.isEmpty(items[operationIndex])) {
- logger.debug("Ignoring Record due to missing Operation - " + Arrays.toString(items));
+ logger.info("Ignoring Record due to missing Operation - " + Arrays.toString(items));
return Result.ignore;
}
if (StringUtils.isEmpty(items[usageQuantityIndex])) {
- logger.debug("Ignoring Record due to missing Usage Quantity - " + Arrays.toString(items));
+ logger.info("Ignoring Record due to missing Usage Quantity - " + Arrays.toString(items));
return Result.ignore;
}
Account account = config.accountService.getAccountById(items[accountIdIndex]);
if (account == null) {
- logger.debug("Ignoring Record due to missing Account - " + Arrays.toString(items));
+ logger.info("Ignoring Record due to missing Account - " + Arrays.toString(items));
return Result.ignore;
}
@@ -182,7 +182,7 @@ else if (product == Product.rds) {
}
if (result == Result.ignore || result == Result.delay) {
- logger.debug("Record not processed - " + result + " - " + Arrays.toString(items));
+ logger.info("Record not processed - " + result + " - " + Arrays.toString(items));
return result;
}
From de1e9f2b3cc61e92499a1529ed2ee4e6e4caf2c9 Mon Sep 17 00:00:00 2001
From: Anthony Johnson <ansoni@gmail.com>
Date: Wed, 21 Oct 2015 08:05:09 -0400
Subject: [PATCH 8/8] Add code to constrain credits to within a month
---
grails-app/conf/BootStrap.groovy | 2 ++
src/java/com/netflix/ice/basic/BasicLineItemProcessor.java | 5 +++++
src/java/com/netflix/ice/common/IceOptions.java | 2 +-
3 files changed, 8 insertions(+), 1 deletion(-)
diff --git a/grails-app/conf/BootStrap.groovy b/grails-app/conf/BootStrap.groovy
index 44e978bf..796bf5b7 100644
--- a/grails-app/conf/BootStrap.groovy
+++ b/grails-app/conf/BootStrap.groovy
@@ -162,6 +162,8 @@ class BootStrap {
properties.setProperty(IceOptions.ONDEMAND_COST_ALERT_THRESHOLD, prop.getProperty(IceOptions.ONDEMAND_COST_ALERT_THRESHOLD));
if (prop.getProperty(IceOptions.URL_PREFIX) != null)
properties.setProperty(IceOptions.URL_PREFIX, prop.getProperty(IceOptions.URL_PREFIX));
+ if (prop.getProperty(IceOptions.IGNORE_CREDITS) != null)
+ properties.setProperty(IceOptions.IGNORE_CREDITS, prop.getProperty(IceOptions.IGNORE_CREDITS));
ReservationCapacityPoller reservationCapacityPoller = null;
if ("true".equals(prop.getProperty("ice.reservationCapacityPoller"))) {
diff --git a/src/java/com/netflix/ice/basic/BasicLineItemProcessor.java b/src/java/com/netflix/ice/basic/BasicLineItemProcessor.java
index d1a0cede..76187ca7 100644
--- a/src/java/com/netflix/ice/basic/BasicLineItemProcessor.java
+++ b/src/java/com/netflix/ice/basic/BasicLineItemProcessor.java
@@ -102,6 +102,7 @@ public Result process(long startMilli, boolean processDelayed, ProcessorConfig c
if (costValue < 0) {
credit = true;
if (config.ignoreCredits || ! reformCredit(startMilli, items)) {
+ logger.info("Ignoring Credit - " + config.ignoreCredits);
return Result.ignore;
}
logger.info("Found Credit - " + Arrays.toString(items));
@@ -208,6 +209,10 @@ else if (result == Result.monthly) {
int numHoursInMonth = new DateTime(startMilli, DateTimeZone.UTC).dayOfMonth().getMaximumValue() * 24;
usageValue = usageValue * endIndex / numHoursInMonth;
costValue = costValue * endIndex / numHoursInMonth;
+ } else {
+ int maxEndIndex = usageData.getNum();
+ if (endIndex > maxEndIndex)
+ endIndex = maxEndIndex;
}
if (monthlyCost) {
diff --git a/src/java/com/netflix/ice/common/IceOptions.java b/src/java/com/netflix/ice/common/IceOptions.java
index 0bf10f92..45961920 100644
--- a/src/java/com/netflix/ice/common/IceOptions.java
+++ b/src/java/com/netflix/ice/common/IceOptions.java
@@ -103,7 +103,7 @@ public class IceOptions {
/**
* Should we ignore credits or not?
*/
- public static final String IGNORE_CREDITS = "ice.ignore_credits";
+ public static final String IGNORE_CREDITS = "ice.ignoreCredits";
/**
* Cost per monitor metric per hour, It's optional.