Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: [~] deprecated methods problem #406

Merged
merged 2 commits into from
Feb 21, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 15 additions & 15 deletions .github/workflows/github-actions-basic.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,15 +29,15 @@ jobs:
SPARK_VERSION: ${{ matrix.spark }}
steps:
- name: Checkout spark-testing-base
uses: actions/checkout@v2
uses: actions/checkout@v4.1.1
# In order to fetch changed files
with:
fetch-depth: 0
repository: holdenk/spark-testing-base
ref: main
# with:
# fetch-depth: 0
# repository: holdenk/spark-testing-base
# ref: main
- name: Cache maven modules
id: cache-maven
uses: actions/cache@v3
uses: actions/cache@v4.0.0
env:
cache-name: cache-maven
with:
Expand All @@ -51,15 +51,15 @@ jobs:
${{ runner.os }}-${{ matrix.spark }}-build-
${{ runner.os }}-${{ matrix.spark }}-
${{ runner.os }}
- name: Sync the current branch with the latest in spark-testing-base
if: github.repository != 'holdenk/spark-testing-base'
id: sync-branch
run: |
apache_spark_ref=`git rev-parse HEAD`
git fetch https://github.com/$GITHUB_REPOSITORY.git ${GITHUB_REF##*/}
git -c user.name='Spark Test Account' -c user.email='[email protected]' merge --no-commit --progress --squash FETCH_HEAD
git -c user.name='Spark Test Account' -c user.email='[email protected]' commit -m "Merged commit"
echo "::set-output name=SPARK_REF::$apache_spark_ref"
# - name: Sync the current branch with the latest in spark-testing-base
# if: github.repository != 'holdenk/spark-testing-base'
# id: sync-branch
# run: |
# apache_spark_ref=`git rev-parse HEAD`
# git fetch https://github.com/$GITHUB_REPOSITORY.git ${GITHUB_REF##*/}
# git -c user.name='Spark Test Account' -c user.email='[email protected]' merge --no-commit --progress --squash FETCH_HEAD
# git -c user.name='Spark Test Account' -c user.email='[email protected]' commit -m "Merged commit"
# echo "::set-output name=SPARK_REF::$apache_spark_ref"
# Cache local repositories. Note that GitHub Actions cache has a 2G limit.
# Run the tests.
- name: Run tests
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -261,21 +261,17 @@ trait DataFrameSuiteBaseLike extends SparkContextProvider
* Compares if two [[DataFrame]]s are equal, checks that the schemas are the same.
* When comparing inexact fields uses tol.
*
* @param tol max acceptable tolerance for numeric (between(0, 1)) &
* timestamp (millis).
* @param customShow unit function to customize the '''show''' method
* when dataframes are not equal. IE: '''df.show(false)''' or
* '''df.toJSON.show(false)'''.
* @param tol max acceptable tolerance for numeric (between(0, 1)) &
* timestamp (millis).
*/
@deprecated(
"Use `assertDataFrameApproximateEquals` with timestamp tolerance",
since = "1.5.0"
)
def assertDataFrameApproximateEquals(
expected: DataFrame, result: DataFrame,
tol: Double, customShow: DataFrame => Unit = _.show()): Unit =
def assertDataFrameApproximateEquals(expected: DataFrame, result: DataFrame,
tol: Double): Unit =
assertDataFrameApproximateEquals(expected, result, tol,
Duration.ofNanos((tol * 1000).toLong), customShow)
Duration.ofMillis(tol.toLong), _.show())

/**
* Compares if two [[DataFrame]]s are equal, checks that the schemas are the same.
Expand Down Expand Up @@ -417,7 +413,7 @@ trait DataFrameSuiteBaseLike extends SparkContextProvider
}

def approxEquals(r1: Row, r2: Row, tol: Double): Boolean = {
DataFrameSuiteBase.approxEquals(r1, r2, tol, Duration.ofNanos((tol*1000).toLong))
DataFrameSuiteBase.approxEquals(r1, r2, tol, Duration.ofMillis(tol.toLong))
}

def approxEquals(r1: Row, r2: Row, tolTimestamp: Duration): Boolean = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,21 +45,16 @@ trait DatasetSuiteBaseLike extends DataFrameSuiteBaseLike {
*
* @param tol max acceptable tolerance for numeric (between(0, 1)) &
* timestamp (millis).
* @param customShow unit function to customize the '''show''' method
* when dataframes are not equal. IE: '''df.show(false)''' or
* '''df.toJSON.show(false)'''.
*/
@deprecated(
"Use `assertDatasetApproximateEquals` with timestamp tolerance",
since = "1.5.0"
)
def assertDatasetApproximateEquals[U]
(expected: Dataset[U], result: Dataset[U], tol: Double,
customShow: DataFrame => Unit = _.show())
def assertDatasetApproximateEquals[U](expected: Dataset[U], result: Dataset[U],
tol: Double)
(implicit UCT: ClassTag[U]): Unit = {

assertDataFrameApproximateEquals(expected.toDF, result.toDF, tol,
Duration.ofNanos((tol * 1000).toLong), customShow)
Duration.ofMillis(tol.toLong), _.show())
}

/**
Expand Down