-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathimport.sh
More file actions
30 lines (25 loc) · 3.33 KB
/
import.sh
File metadata and controls
30 lines (25 loc) · 3.33 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
#!/usr/bin/env bash
for d in 20180102 20180103 20180104 20180105 20180108 20180109 20180110; do
hbase org.apache.hadoop.hbase.mapreduce.ImportTsv \
-Dimporttsv.separator=, \
-Dimporttsv.columns="HBASE_ROW_KEY,data:uiPreClosePx,data:uiOpenPx,data:uiHighPx,data:uiLowPx,data:uiLastPx,data:uiNumTrades,data:strInstrumentStatus,data:uiTotalVolumeTrade,data:uiTotalValueTrade,data:uiTotalBidQty,data:uiTotalOfferQty,data:uiWeightedAvgBidPx,data:uiWeightedAvgOfferPx,data:uiWithdrawBuyNumber,data:uiWithdrawSellNumber,data:uiWithdrawBuyAmount,data:uiWithdrawBuyMoney,data:uiWithdrawSellAmount,data:uiWithdrawSellMoney,data:uiTotalBidNumber,data:uiTotalOfferNumber,data:uiBidTradeMaxDuration,data:uiOfferTradeMaxDuration,data:uiNumBidOrders,data:uiNumOfferOrders,data:arrBidPrice,data:arrBidOrderQty,data:arrBidNumOrders,data:arrBidOrders,data:arrOfferPrice,data:arrOfferOrderQty,data:arrOfferNumOrders,data:arrOfferOrders,data:uiIOPV,data:uiETFBuyNumber,data:uiETFBuyAmount,data:uiETFBuyMoney,data:uiETFSellNumber,data:uiETFSellAmount,data:uiETFSellMoney" \
-Dimporttsv.bulk.output=h_${d} \
hisdata30g \
hdfs://srv1:9000/30Gdata/${d}/data.csv
done
hbase org.apache.hadoop.hbase.mapreduce.ImportTsv \
-Dimporttsv.separator=, \
-Dimporttsv.columns="HBASE_ROW_KEY,data:uiPreClosePx,data:uiOpenPx,data:uiHighPx,data:uiLowPx,data:uiLastPx,data:uiNumTrades,data:strInstrumentStatus,data:uiTotalVolumeTrade,data:uiTotalValueTrade,data:uiTotalBidQty,data:uiTotalOfferQty,data:uiWeightedAvgBidPx,data:uiWeightedAvgOfferPx,data:uiWithdrawBuyNumber,data:uiWithdrawSellNumber,data:uiWithdrawBuyAmount,data:uiWithdrawBuyMoney,data:uiWithdrawSellAmount,data:uiWithdrawSellMoney,data:uiTotalBidNumber,data:uiTotalOfferNumber,data:uiBidTradeMaxDuration,data:uiOfferTradeMaxDuration,data:uiNumBidOrders,data:uiNumOfferOrders,data:arrBidPrice,data:arrBidOrderQty,data:arrBidNumOrders,data:arrBidOrders,data:arrOfferPrice,data:arrOfferOrderQty,data:arrOfferNumOrders,data:arrOfferOrders,data:uiIOPV,data:uiETFBuyNumber,data:uiETFBuyAmount,data:uiETFBuyMoney,data:uiETFSellNumber,data:uiETFSellAmount,data:uiETFSellMoney" \
-Dimporttsv.bulk.output=h_20180102 \
hisdata30g \
hdfs://srv1:9000/30Gdata/20180102/data.csv
hadoop jar lib/hbase-server-1.4.8.jar completebulkload h_20180102 hisdata30g
hbase org.apache.hadoop.hbase.mapreduce.ImportTsv \
-Dimporttsv.separator=, \
-Dimporttsv.columns="HBASE_ROW_KEY,data:uiPreClosePx,data:uiOpenPx,data:uiHighPx,data:uiLowPx,data:uiLastPx,data:uiNumTrades,data:strInstrumentStatus,data:uiTotalVolumeTrade,data:uiTotalValueTrade,data:uiTotalBidQty,data:uiTotalOfferQty,data:uiWeightedAvgBidPx,data:uiWeightedAvgOfferPx,data:uiWithdrawBuyNumber,data:uiWithdrawSellNumber,data:uiWithdrawBuyAmount,data:uiWithdrawBuyMoney,data:uiWithdrawSellAmount,data:uiWithdrawSellMoney,data:uiTotalBidNumber,data:uiTotalOfferNumber,data:uiBidTradeMaxDuration,data:uiOfferTradeMaxDuration,data:uiNumBidOrders,data:uiNumOfferOrders,data:arrBidPrice,data:arrBidOrderQty,data:arrBidNumOrders,data:arrBidOrders,data:arrOfferPrice,data:arrOfferOrderQty,data:arrOfferNumOrders,data:arrOfferOrders,data:uiIOPV,data:uiETFBuyNumber,data:uiETFBuyAmount,data:uiETFBuyMoney,data:uiETFSellNumber,data:uiETFSellAmount,data:uiETFSellMoney" \
hisdata30g \
hdfs://srv1:9000/test.csv
dd if=/dev/zero of=/mds/swap count=16384 bs=1MiB
sudo chmod 600 /mds/swap
sudo mkswap /mds/swap
sudo swapon /mds/swap