Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add Support for Credits/Refunds #158

Open
wants to merge 8 commits into
base: master
Choose a base branch
from
Open

Add Support for Credits/Refunds #158

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
72c6169
first pass for credits
ansoni Feb 26, 2015
e390873
-- re-work some of the credit handling
ansoni Mar 2, 2015
33386d8
Revert accidental modifications pulled in from another feature branch
ansoni Mar 2, 2015
30e466c
small cleanup
ansoni Mar 2, 2015
89aba5e
Updates to credit processing to ensure we capture all credit entries …
ansoni Apr 9, 2015
3c315c0
Add Support to turn Credit Support On/Off
ansoni Jul 13, 2015
6283e94
Move debug to info for ignored and credit line-items
ansoni Jul 13, 2015
de1e9f2
Add code to constrain credits to within a month
ansoni Oct 21, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions grails-app/conf/BootStrap.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,8 @@ class BootStrap {
properties.setProperty(IceOptions.ONDEMAND_COST_ALERT_THRESHOLD, prop.getProperty(IceOptions.ONDEMAND_COST_ALERT_THRESHOLD));
if (prop.getProperty(IceOptions.URL_PREFIX) != null)
properties.setProperty(IceOptions.URL_PREFIX, prop.getProperty(IceOptions.URL_PREFIX));
if (prop.getProperty(IceOptions.IGNORE_CREDITS) != null)
properties.setProperty(IceOptions.IGNORE_CREDITS, prop.getProperty(IceOptions.IGNORE_CREDITS));

ReservationCapacityPoller reservationCapacityPoller = null;
if ("true".equals(prop.getProperty("ice.reservationCapacityPoller"))) {
Expand Down
122 changes: 103 additions & 19 deletions src/java/com/netflix/ice/basic/BasicLineItemProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.Arrays;
import java.util.List;
import java.util.Map;

Expand Down Expand Up @@ -88,23 +89,60 @@ public long getEndMillis(String[] items) {
}

public Result process(long startMilli, boolean processDelayed, ProcessorConfig config, String[] items, Map<Product, ReadWriteData> usageDataByProduct, Map<Product, ReadWriteData> costDataByProduct, Map<String, Double> ondemandRate) {
if (StringUtils.isEmpty(items[accountIdIndex]) ||
StringUtils.isEmpty(items[productIndex]) ||
StringUtils.isEmpty(items[usageTypeIndex]) ||
StringUtils.isEmpty(items[operationIndex]) ||
StringUtils.isEmpty(items[usageQuantityIndex]) ||
StringUtils.isEmpty(items[costIndex]))

if (StringUtils.isEmpty(items[costIndex])) {
logger.info("Ignoring Record due to missing Cost - " + Arrays.toString(items));
return Result.ignore;
}

double costValue = Double.parseDouble(items[costIndex]);
boolean credit = false;

// make sure we don't ignore credits
if (costValue < 0) {
credit = true;
if (config.ignoreCredits || ! reformCredit(startMilli, items)) {
logger.info("Ignoring Credit - " + config.ignoreCredits);
return Result.ignore;
}
logger.info("Found Credit - " + Arrays.toString(items));
}

// fail-fast on records we can't process
if (StringUtils.isEmpty(items[accountIdIndex])) {
logger.info("Ignoring Record due to missing Account Id - " + Arrays.toString(items));
return Result.ignore;
}
if (StringUtils.isEmpty(items[productIndex])) {
logger.info("Ignoring Record due to missing Product - " + Arrays.toString(items));
return Result.ignore;
}
if (StringUtils.isEmpty(items[usageTypeIndex])) {
logger.info("Ignoring Record due to missing Usage Type - " + Arrays.toString(items));
return Result.ignore;
}
if (StringUtils.isEmpty(items[operationIndex])) {
logger.info("Ignoring Record due to missing Operation - " + Arrays.toString(items));
return Result.ignore;
}
if (StringUtils.isEmpty(items[usageQuantityIndex])) {
logger.info("Ignoring Record due to missing Usage Quantity - " + Arrays.toString(items));
return Result.ignore;
}

Account account = config.accountService.getAccountById(items[accountIdIndex]);
if (account == null)
if (account == null) {
logger.info("Ignoring Record due to missing Account - " + Arrays.toString(items));
return Result.ignore;
}

double usageValue = Double.parseDouble(items[usageQuantityIndex]);
double costValue = Double.parseDouble(items[costIndex]);

long millisStart;
long millisEnd;

Result result = Result.hourly;

try {
millisStart = amazonBillingDateFormat.parseMillis(items[startTimeIndex]);
millisEnd = amazonBillingDateFormat.parseMillis(items[endTimeIndex]);
Expand All @@ -116,7 +154,7 @@ public Result process(long startMilli, boolean processDelayed, ProcessorConfig c

Product product = config.productService.getProductByAwsName(items[productIndex]);
boolean reservationUsage = "Y".equals(items[reservedIndex]);
ReformedMetaData reformedMetaData = reform(millisStart, config, product, reservationUsage, items[operationIndex], items[usageTypeIndex], items[descriptionIndex], costValue);
ReformedMetaData reformedMetaData = reform(millisStart, config, product, reservationUsage, items[operationIndex], items[usageTypeIndex], items[descriptionIndex], costValue, credit);
product = reformedMetaData.product;
Operation operation = reformedMetaData.operation;
UsageType usageType = reformedMetaData.usageType;
Expand All @@ -125,7 +163,6 @@ public Result process(long startMilli, boolean processDelayed, ProcessorConfig c
int startIndex = (int)((millisStart - startMilli)/ AwsUtils.hourMillis);
int endIndex = (int)((millisEnd + 1000 - startMilli)/ AwsUtils.hourMillis);

Result result = Result.hourly;
if (product == Product.ec2_instance) {
result = processEc2Instance(processDelayed, reservationUsage, operation, zone);
}
Expand All @@ -145,13 +182,15 @@ else if (product == Product.rds) {
result = processRds(usageType);
}

if (result == Result.ignore || result == Result.delay)
if (result == Result.ignore || result == Result.delay) {
logger.info("Record not processed - " + result + " - " + Arrays.toString(items));
return result;
}

if (usageType.name.startsWith("TimedStorage-ByteHrs"))
result = Result.daily;

boolean monthlyCost = StringUtils.isEmpty(items[descriptionIndex]) ? false : items[descriptionIndex].toLowerCase().contains("-month");
boolean monthlyCost = StringUtils.isEmpty(items[descriptionIndex]) ? false : ( items[descriptionIndex].toLowerCase().contains("-month") );

ReadWriteData usageData = usageDataByProduct.get(null);
ReadWriteData costData = costDataByProduct.get(null);
Expand All @@ -170,6 +209,10 @@ else if (result == Result.monthly) {
int numHoursInMonth = new DateTime(startMilli, DateTimeZone.UTC).dayOfMonth().getMaximumValue() * 24;
usageValue = usageValue * endIndex / numHoursInMonth;
costValue = costValue * endIndex / numHoursInMonth;
} else {
int maxEndIndex = usageData.getNum();
if (endIndex > maxEndIndex)
endIndex = maxEndIndex;
}

if (monthlyCost) {
Expand Down Expand Up @@ -213,7 +256,7 @@ else if (result == Result.monthly) {
}
catch (Exception e) {
logger.error("failed to get RI price for " + tagGroup.region + " " + usageTypeForPrice);
resourceCostValue = -1;
resourceCostValue = Double.MIN_VALUE;
}
}

Expand All @@ -234,7 +277,9 @@ else if (result == Result.monthly) {
return result;

for (int i : indexes) {

if (credit) {
logger.debug("Handled Credit Index " + i + " - " + costValue);
}
if (config.randomizer != null) {

if (tagGroup.product != Product.rds && tagGroup.product != Product.s3 && usageData.getData(i).get(tagGroup) != null)
Expand All @@ -248,8 +293,8 @@ else if (result == Result.monthly) {
Map<TagGroup, Double> usages = usageData.getData(i);
Map<TagGroup, Double> costs = costData.getData(i);

addValue(usages, tagGroup, usageValue, config.randomizer == null || tagGroup.product == Product.rds || tagGroup.product == Product.s3);
addValue(costs, tagGroup, costValue, config.randomizer == null || tagGroup.product == Product.rds || tagGroup.product == Product.s3);
addValue(usages, tagGroup, usageValue, config.randomizer == null || tagGroup.product == Product.rds || tagGroup.product == Product.s3 || credit == true);
addValue(costs, tagGroup, costValue, config.randomizer == null || tagGroup.product == Product.rds || tagGroup.product == Product.s3 || credit == true);
}
else {
resourceCostValue = usageValue * config.costPerMonitorMetricPerHour;
Expand All @@ -259,9 +304,9 @@ else if (result == Result.monthly) {
Map<TagGroup, Double> usagesOfResource = usageDataOfProduct.getData(i);
Map<TagGroup, Double> costsOfResource = costDataOfProduct.getData(i);

if (config.randomizer == null || tagGroup.product == Product.rds || tagGroup.product == Product.s3) {
if (config.randomizer == null || tagGroup.product == Product.rds || tagGroup.product == Product.s3 || credit == true) {
addValue(usagesOfResource, resourceTagGroup, usageValue, product != Product.monitor);
if (!config.useCostForResourceGroup.equals("modeled") || resourceCostValue < 0) {
if (!config.useCostForResourceGroup.equals("modeled") || resourceCostValue == Double.MIN_VALUE) {
addValue(costsOfResource, resourceTagGroup, costValue, product != Product.monitor);
} else {
addValue(costsOfResource, resourceTagGroup, resourceCostValue, product != Product.monitor);
Expand Down Expand Up @@ -345,7 +390,42 @@ private Result processRds(UsageType usageType) {
return Result.hourly;
}

protected ReformedMetaData reform(long millisStart, ProcessorConfig config, Product product, boolean reservationUsage, String operationStr, String usageTypeStr, String description, double cost) {
protected boolean reformCredit(long startMilli, String[] items) {

String[] split_description = items[descriptionIndex].split(":");

// If the credit doesn't have a start and end time then we... set to end to end of month
if (items[startTimeIndex].isEmpty()) {
int numHoursInMonth = new DateTime(startMilli, DateTimeZone.UTC).dayOfMonth().getMaximumValue() * 24;
items[startTimeIndex]=new DateTime(startMilli, DateTimeZone.UTC).plusHours(numHoursInMonth-1).toString(amazonBillingDateFormat);
items[endTimeIndex]=new DateTime(startMilli, DateTimeZone.UTC).plusHours(numHoursInMonth).toString(amazonBillingDateFormat);
logger.debug("Credit did not have a Time - Set to " + items[startTimeIndex] + "-" + items[endTimeIndex]) ;
}

// seperate credits into their own product for easy filtering/aggregation
items[productIndex]+=" credit";

// try to use the description to fetch some info about the credit if we have nothing
if (items[operationIndex].isEmpty()) {
if (split_description.length > 0)
items[operationIndex]=split_description[0];
else
items[operationIndex]="credit";
}

if (items[usageTypeIndex].isEmpty()) {
items[usageTypeIndex]="credit";
}

if (items[usageQuantityIndex].isEmpty()) {
items[usageQuantityIndex] = "1";
}

return true;

}

protected ReformedMetaData reform(long millisStart, ProcessorConfig config, Product product, boolean reservationUsage, String operationStr, String usageTypeStr, String description, double cost, boolean credit) {

Operation operation = null;
UsageType usageType = null;
Expand Down Expand Up @@ -432,6 +512,10 @@ else if (usageTypeStr.startsWith("HeavyUsage") || usageTypeStr.startsWith("Mediu
usageType = UsageType.getUsageType(usageTypeStr, operation, description);
}

// This method resets product. Make sure we seperated out our credits
if (credit && ! product.name.endsWith("credit"))
product = new Product(product.name + " credit");

return new ReformedMetaData(region, product, operation, usageType);
}

Expand Down
5 changes: 5 additions & 0 deletions src/java/com/netflix/ice/common/IceOptions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,11 @@ public class IceOptions {
*/
public static final String MONTHLY_CACHE_SIZE = "ice.monthlycachesize";

/**
* Should we ignore credits or not?
*/
public static final String IGNORE_CREDITS = "ice.ignoreCredits";

/**
* Cost per monitor metric per hour, It's optional.
*/
Expand Down
2 changes: 2 additions & 0 deletions src/java/com/netflix/ice/processor/ProcessorConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ public class ProcessorConfig extends Config {
public final LineItemProcessor lineItemProcessor;
public final Randomizer randomizer;
public final double costPerMonitorMetricPerHour;
public final boolean ignoreCredits;

public final String useCostForResourceGroup;

Expand Down Expand Up @@ -87,6 +88,7 @@ public ProcessorConfig(
customTags = properties.getProperty(IceOptions.CUSTOM_TAGS, "").split(",");

useCostForResourceGroup = properties.getProperty(IceOptions.RESOURCE_GROUP_COST, "modeled");
ignoreCredits = Boolean.parseBoolean(properties.getProperty(IceOptions.IGNORE_CREDITS, "true"));

ProcessorConfig.instance = this;

Expand Down
4 changes: 2 additions & 2 deletions web-app/js/ice.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ ice.factory('highchart', function() {
}

var setupYAxis = function(isCost, showsps, factorsps) {
var yAxis = {title:{text: (isCost ? 'Cost' : 'Usage') + " per " + (factorsps ? metricunitname : consolidate)}, min: 0, lineWidth: 2};
var yAxis = {title:{text: (isCost ? 'Cost' : 'Usage') + " per " + (factorsps ? metricunitname : consolidate)}, lineWidth: 2};
if (isCost)
yAxis.labels = {
formatter: function() {
Expand All @@ -169,7 +169,7 @@ ice.factory('highchart', function() {
hc_options.yAxis = [yAxis];

if (showsps) {
hc_options.yAxis.push({title:{text:metricname}, height: 100, min: 0, lineWidth: 2, offset: 0});
hc_options.yAxis.push({title:{text:metricname}, height: 100, lineWidth: 2, offset: 0});
hc_options.yAxis[0].top = 150;
hc_options.yAxis[0].height = 350;
}
Expand Down