data_wrangling2.Rmd

---
title: "R Notebook"
output: html_notebook
---


```{r include=FALSE}
library(lubridate)
library(zoo)
library(readr)
library(caret)
library(forecast)
library(AppliedPredictiveModeling)
library(e1071)
library(TTR)
library(DMwR)
library(neuralnet)
library(dplyr)
```

```{r}
df <- read.csv("./data/btc/btc-final.csv", sep = '\t')
```

# Calculate 5-min periods in 7, 14, and 28 days
```{r}
days7 <- 60/5*24*7
days14 <- 60/5*24*14
days28 <- 60/5*24*28
```

# Make data.frame for new features
```{r}
new_features <- data.frame(date = df$date)
```

# Volatility 
```{r}
# 7 day
ohlc <- df[,c("price_open", "price_high", "price_low", "price_close")]
new_features$Volatilityclose7 <- volatility(ohlc, n = days7, calc="close") #based on close only
new_features$Volatilitygkyz7 <- volatility(ohlc, n = days7, calc="gk.yz") #based on open, high, low, and close
new_features$Volatilityparkinson7 <- volatility(ohlc, n = days7, calc="parkinson") #based on high/low

ohlc2 <- df[,c("close_price_returns", "price_high", "price_low", "price_close")]
new_features$Volatilityreturns7 <- volatility(ohlc2, n = days7, calc="close", mean0=TRUE)

# 40 period (200 min, 3.3333 hours)
ohlc <- df[,c("price_open", "price_high", "price_low", "price_close")]
new_features$Volatilityclose40 <- volatility(ohlc, n = 40, calc="close") #based on close only
new_features$Volatilitygkyz40 <- volatility(ohlc, n = 40, calc="gk.yz") #based on open, high, low, and close
new_features$Volatilityparkinson40 <- volatility(ohlc, n = 40, calc="parkinson") #based on high/low

ohlc2 <- df[,c("close_price_returns", "price_high", "price_low", "price_close")]
new_features$Volatilityreturns40 <- volatility(ohlc2, n = 40, calc="close", mean0=TRUE)
```

# Moving Averages
```{r}
#Smooth Moving Average
new_features$SMA7 <- SMA(df$price_close, n=days7)
new_features$SMA14 <- SMA(df$price_close, n=days14)
new_features$SMA28 <- SMA(df$price_close, n=days28)

#Exponential Moving Average
new_features$EMA7 <- EMA(df$price_close, n=days7)
new_features$EMA14 <- EMA(df$price_close, n=days14)
new_features$EMA28 <- EMA(df$price_close, n=days28)

#Volume SMA and EMA
new_features$VolSMA7 <- SMA(df$volume, n=days7)
new_features$VolSMA14 <- SMA(df$volume, n=days14)
new_features$VolSMA28 <- SMA(df$volume, n=days28)
new_features$VolEMA7 <- EMA(df$volume, n=days7)
new_features$VolEMA14 <- EMA(df$volume, n=days14)
new_features$VolEMA28 <- EMA(df$volume, n=days28)
```

# SAR (Stop-and-Reverse)
```{r}
hl <- df[,c("price_high", "price_low")]
new_features$SAR <- SAR(hl)
```

# MACD
```{r}
#for 5 minute periods
nFast = 12
nSlow = 26
nSig = 9
MACD5min <- as.data.frame(MACD(df[,"price_close"], nFast=nFast, nSlow=nSlow, nSig=nSig, maType=EMA, percent=TRUE ))
new_features$MACD5min <- MACD5min$macd
new_features$MACD5minSignal <- MACD5min$signal

# for days
nFast = 12*(60/5*24) #12 days fast MA
nSlow = 26*(60/5*24) #26 days slow MA
nSig = 9*(60/5*24) #9 days signal moving average
MACDday <- as.data.frame(MACD(df[,"price_close"], nFast=nFast, nSlow=nSlow, nSig=nSig, maType=EMA, percent=TRUE ))
new_features$MACDday <- MACDday$macd
new_features$MACDdaySignal <- MACDday$signal

# for volume
MACDvolume <- as.data.frame(MACD(df[,"volume"], nFast=nFast, nSlow=nSlow, nSig=nSig, maType=EMA, percent=FALSE ))
new_features$MACDvolume <- MACDvolume$macd
```

# ATR (Average True Range)
```{r}
hlc <- df[,c("price_high", "price_low", "price_close")]
atr <- as.data.frame(ATR(hlc, n=days7))
new_features$ATR <- atr$atr
atr <- atr$atr/atr$trueLow
new_features$ATRscaled <- atr
```

# RSI (Relative Strength Index)
```{r}
new_features$RSI <- RSI(df$price_close, n=days14)
new_features$RSI40 <- RSI(df$price_close, n=40)
```

# OBV (On Balance Volume)
```{r}
# OBV(df$price_close, df$volume)[100000:101000]
# OBV(df$price_close, df$volume)[300000:301000]
```

# 40 Period Windows
```{r}
training_windows <- df %>% 
  mutate(id = 1:nrow(df)) %>%
  select(id, price_close, close_price_returns, volume, up, SinceUP, SinceDown) 
training_windows$Volalility <- new_features$Volatilitygkyz40
training_windows$VolalilityReturns <- new_features$Volatilityreturns40

other_vars <- df %>% select(y_returns = yret, y_close_price = yprice) %>%
  cbind(SAR = new_features$SAR, MACD = new_features$MACDday, ATRScaled = new_features$ATRscaled, 
        RSI = new_features$RSI, RSI40 = new_features$RSI40)
```

```{r}
z <- read.zoo(training_windows)
zz <- zoo:::lag.zoo(z, 1:40) 

z2 <- as.data.frame(z)
zz2 <- as.data.frame(zz)

zz2 <- rbind(zz2, zz2[1,]) #add and extra row to the beginning of the lags to make it the same lengths as other data.frames

nrow(df)
nrow(z2)
nrow(zz2)
nrow(other_vars)


training_windows <- cbind(date = df$date, z2, zz2, other_vars)
training_windows <- training_windows[-nrow(training_windows), ] #remove the incorrect final row
```


```{r}
training_windows$date <- as.POSIXct(training_windows$date)
```


```{r}
is.nan.data.frame <- function(x) do.call(cbind, lapply(x, is.nan))

tmp <- training_windows %>% filter(date > "2015-01-27")
tmp2 <- tmp[!complete.cases(tmp),]


sapply(tmp2, function(x){sum(is.nan(x))}) %>% max

tmp2[is.nan(tmp2)] <- 0

sapply(tmp2, function(x){sum(is.nan(x))}) %>% max
```


```{r}
write_tsv(training_windows, "data/btc/training_windows.csv")
# write.table(training_windows, "data/btc/training_windows.csv", sep='\t')
```

```{r}
write.table(new_features, "data/btc/indicators.csv", sep='\t')
```