Skip to content
This repository was archived by the owner on Aug 29, 2023. It is now read-only.

modify apply patch scripts for feature enable QATCodec on sparkSql pa… #22

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

modify apply patch scripts for feature enable QATCodec on sparkSql pa… #22

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 33 additions & 29 deletions columnar_format_qat_wrapper/apply_hive_jars.sh
100644 → 100755
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,11 @@
# */
#!/bin/bash

declare -a supported_CDH_versions=("5.14.2" "6.2.1" "7.0.0")
declare -A cdh_parquet_format_version_m=( ["5.14.2"]="2.1.0" ["7.0.0"]="2.4.0")
declare -A cdh_parquet_mr_version_m=( ["5.14.2"]="1.5.0" ["7.0.0"]="1.10.0")
declare -A cdp_orc_version_m=(["7.0.0"]="1.5.1")
declare -A cdh_hive_version_m=( ["5.14.2"]="1.1.0" ["7.0.0"]="3.1.0")
declare -a supported_CDH_versions=("5.14.2" "6.2.1" "7.0.0" "spark3.0.0")
declare -A cdh_parquet_format_version_m=( ["5.14.2"]="2.1.0" ["7.0.0"]="2.4.0" ["spark3.0.0"]="2.4.0")
declare -A cdh_parquet_mr_version_m=( ["5.14.2"]="1.5.0" ["7.0.0"]="1.10.0" ["spark3.0.0"]="1.10.1")
declare -A cdp_orc_version_m=(["7.0.0"]="1.5.1" ["spark3.0.0"]="1.5.10")
declare -A cdh_hive_version_m=( ["5.14.2"]="1.1.0" ["7.0.0"]="3.1.0" ["spark3.0.0"]="2.3.7")
# Repo Address
PARQUET_MR_REPO=https://github.com/cloudera/parquet-mr
PARQUET_FORMAT_REPO=https://github.com/cloudera/parquet-format
Expand Down Expand Up @@ -69,14 +69,16 @@ function check_CDH_version(){
}

apply_patch_to_cdp_orc(){
pushd $TARGET_DIR
CDH_major_version=$(echo $CDH_release_version | cut -d '.' -f 1)
if [ "$CDH_major_version" = "7" ]; then
ORC_BRANCH="rel/release-${cdp_orc_version_m[$CDH_release_version]}"
if [ "$CDH_major_version" != "spark3" ]; then
pushd $TARGET_DIR
if [ "$CDH_major_version" = "7" ]; then
ORC_BRANCH="rel/release-${cdp_orc_version_m[$CDH_release_version]}"
fi
clone_repo $ORC_BRANCH $UPSTREAM_ORC_REPO
popd
apply_diff_to_orc
fi
clone_repo $ORC_BRANCH $UPSTREAM_ORC_REPO
popd
apply_diff_to_orc
}

apply_diff_to_orc(){
Expand All @@ -89,22 +91,24 @@ apply_diff_to_orc(){
}

apply_patch_to_cdh_hive(){
pushd $TARGET_DIR
CDH_major_version=$(echo $CDH_release_version | cut -d '.' -f 1)
if [ "$CDH_major_version" = "6" ]; then
HIVE_BRANCH="cdh$CDH_release_version"
elif [ "$CDH_major_version" = "7" ]; then
HIVE_BRANCH="rel/release-${cdh_hive_version_m[$CDH_release_version]}"
clone_repo $HIVE_BRANCH $UPSTREAM_HIVE_REPO
popd
apply_diff_to_hive
return
else
HIVE_BRANCH="cdh$CDH_major_version-${cdh_hive_version_m[$CDH_release_version]}_$CDH_release_version"
if [ "$CDH_major_version" != "spark3" ]; then
pushd $TARGET_DIR
if [ "$CDH_major_version" = "6" ]; then
HIVE_BRANCH="cdh$CDH_release_version"
elif [ "$CDH_major_version" = "7" ]; then
HIVE_BRANCH="rel/release-${cdh_hive_version_m[$CDH_release_version]}"
clone_repo $HIVE_BRANCH $UPSTREAM_HIVE_REPO
popd
apply_diff_to_hive
return
else
HIVE_BRANCH="cdh$CDH_major_version-${cdh_hive_version_m[$CDH_release_version]}_$CDH_release_version"
fi
clone_repo $HIVE_BRANCH $HIVE_REPO
echo yes | cp -rf $HIVE_QAT_DIR/$CDH_release_version/hive $TARGET_DIR/
popd
fi
clone_repo $HIVE_BRANCH $HIVE_REPO
echo yes | cp -rf $HIVE_QAT_DIR/$CDH_release_version/hive $TARGET_DIR/
popd
}

apply_diff_to_hive(){
Expand All @@ -121,7 +125,7 @@ apply_patch_to_cdh_parquet_format(){
CDH_major_version=$(echo $CDH_release_version | cut -d '.' -f 1)
if [ "$CDH_major_version" = "6" ]; then
PARQUET_FORMAT_BRANCH="cdh$CDH_release_version"
elif [ "$CDH_major_version" = "7" ]; then
elif [ "$CDH_major_version" = "7" ] || [ "$CDH_major_version" = "spark3" ]; then
PARQUET_FORMAT_BRANCH="apache-parquet-format-${cdh_parquet_format_version_m[$CDH_release_version]}"
clone_repo $PARQUET_FORMAT_BRANCH $UPSTREAM_PARQUET_FORMAT_REPO
popd
Expand All @@ -137,7 +141,7 @@ apply_patch_to_cdh_parquet_format(){

apply_diff_to_parquet_format(){
CDH_major_version=$(echo $CDH_release_version | cut -d '.' -f 1)
if [ "$CDH_major_version" = "7" ]; then
if [ "$CDH_major_version" = "7" ] || [ "$CDH_major_version" = "spark3" ]; then
pushd $PARQUET_FORMAT_SRC_DIR
git apply --reject --whitespace=fix $HIVE_QAT_DIR/$CDH_release_version/parquet-format/*.diff
popd
Expand All @@ -149,7 +153,7 @@ apply_patch_to_cdh_parquet_mr(){
CDH_major_version=$(echo $CDH_release_version | cut -d '.' -f 1)
if [ "$CDH_major_version" = "6" ]; then
PARQUET_MR_BRANCH="cdh$CDH_release_version"
elif [ "$CDH_major_version" = "7" ]; then
elif [ "$CDH_major_version" = "7" ] || [ "$CDH_major_version" = "spark3" ]; then
PARQUET_MR_BRANCH="apache-parquet-${cdh_parquet_mr_version_m[$CDH_release_version]}"
clone_repo $PARQUET_MR_BRANCH $UPSTREAM_PARQUET_MR_REPO
popd
Expand All @@ -165,7 +169,7 @@ apply_patch_to_cdh_parquet_mr(){

apply_diff_to_parquet_mr(){
CDH_major_version=$(echo $CDH_release_version | cut -d '.' -f 1)
if [ "$CDH_major_version" = "7" ]; then
if [ "$CDH_major_version" = "7" ] || [ "$CDH_major_version" = "spark3" ]; then
pushd $PARQUET_MR_SRC_DIR
git apply --reject --whitespace=fix $HIVE_QAT_DIR/$CDH_release_version/parquet-mr/*.diff
popd
Expand Down
Loading