From f7ad6f45126194e9fcf8f0194970c6c8e219f31b Mon Sep 17 00:00:00 2001 From: Fabian Gans Date: Mon, 29 Mar 2021 11:51:09 +0200 Subject: [PATCH 1/2] Add CfGRIB dataset interface --- src/YAXArrayBase.jl | 3 +-- src/datasets/grib.jl | 49 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 src/datasets/grib.jl diff --git a/src/YAXArrayBase.jl b/src/YAXArrayBase.jl index fc6b551..10f0b3f 100644 --- a/src/YAXArrayBase.jl +++ b/src/YAXArrayBase.jl @@ -36,8 +36,7 @@ function __init__() @require NetCDF="30363a11-5582-574a-97bb-aa9a979735b9" include("datasets/netcdf.jl") - - + @require CfGRIB="b476ad61-2aff-463c-b862-1d4433a0d503" include("datasets/grib.jl") end diff --git a/src/datasets/grib.jl b/src/datasets/grib.jl new file mode 100644 index 0000000..cb0248d --- /dev/null +++ b/src/datasets/grib.jl @@ -0,0 +1,49 @@ +using .CfGRIB: CfGRIB +using DiskArrays: AbstractDiskArray, DiskArrays, GridChunks +# Define a DiskArray version of OnDiskArray, which is actually a subtype of AbstractArray +struct CFDiskVariable{T,N} <: AbstractDiskArray{T,N} + a::CfGRIB.OnDiskArray + s::NTuple{N,Int} + cs::GridChunks{N} +end +function CFDiskVariable(d::CfGRIB.OnDiskArray) + s = size(d) + N = length(s) + cs = reverse(DiskArrays.estimate_chunksize(reverse(s), sizeof(d.dtype))) + CFDiskVariable{d.dtype,N}(d, s, GridChunks(s,cs)) +end +Base.size(v::CFDiskVariable) = v.s +DiskArrays.eachchunk(b::CFDiskVariable) = b.cs +DiskArrays.haschunks(::CFDiskVariable) = DiskArrays.Chunked() +function DiskArrays.readblock!(b::CFDiskVariable,aout,r::AbstractUnitRange...) + res = b.a[r...] + aout .= reshape(res, size(aout)) + nothing +end + +#And implement the Dataset interface +struct CfGRIBDataset + ds::CfGRIB.DataSet +end +YAXArrayBase.to_dataset(::Type{CfGRIBDataset},p::String) = CfGRIBDataset(CfGRIB.DataSet(p)) +toarray(a::CfGRIB.OnDiskArray) = CFDiskVariable(a) +toarray(a) = a +toarray(a::Number) = fill(a) +function YAXArrayBase.get_var_handle(ds::CfGRIBDataset, name) + toarray(ds.ds.variables[string(name)].data) +end +Base.haskey(ds::CfGRIBDataset, k) = haskey(ds.ds.variables, k) + +YAXArrayBase.get_varnames(ds::CfGRIBDataset) = collect(keys(ds.ds.variables)) + +YAXArrayBase.get_var_dims(ds::CfGRIBDataset, k) = ds.ds.variables[k].dimensions + +function YAXArrayBase.get_var_attrs(ds::CfGRIBDataset,name) + ds.ds.variables[string(name)].attributes +end + +using YAXArrayBase: backendlist, backendregex +backendlist[:GRIB] = CfGRIBDataset +push!(backendregex,r".grb$"=>CfGRIBDataset) +push!(backendregex,r".grib$"=>CfGRIBDataset); + From 532c6a1a0d398d38cbe3c6c4837251cd07ecfdee Mon Sep 17 00:00:00 2001 From: Fabian Gans Date: Mon, 29 Mar 2021 11:56:14 +0200 Subject: [PATCH 2/2] update CI --- .codecov.yml | 1 - .github/workflows/CI.yml | 49 ++++++++++++++++++++++++++++++ .github/workflows/CompatHelper.yml | 24 +++++++++++++++ .github/workflows/TagBot.yml | 7 +++-- .travis.yml | 30 ------------------ appveyor.yml | 43 -------------------------- 6 files changed, 78 insertions(+), 76 deletions(-) delete mode 100644 .codecov.yml create mode 100644 .github/workflows/CI.yml create mode 100644 .github/workflows/CompatHelper.yml delete mode 100644 .travis.yml delete mode 100644 appveyor.yml diff --git a/.codecov.yml b/.codecov.yml deleted file mode 100644 index 69cb760..0000000 --- a/.codecov.yml +++ /dev/null @@ -1 +0,0 @@ -comment: false diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml new file mode 100644 index 0000000..27b5843 --- /dev/null +++ b/.github/workflows/CI.yml @@ -0,0 +1,49 @@ +name: CI +on: + pull_request: + push: + branches: + - master + tags: '*' +jobs: + test: + name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + version: + - '1' + - 'nightly' + os: + - ubuntu-latest + - macOS-latest + - windows-latest + arch: + - x64 + steps: + - uses: actions/checkout@v2 + - uses: julia-actions/setup-julia@v1 + with: + version: ${{ matrix.version }} + arch: ${{ matrix.arch }} + - uses: actions/cache@v1 + env: + cache-name: cache-artifacts + with: + path: ~/.julia/artifacts + key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }} + restore-keys: | + ${{ runner.os }}-test-${{ env.cache-name }}- + ${{ runner.os }}-test- + ${{ runner.os }}- + - uses: julia-actions/julia-buildpkg@v1 + env: + PYTHON: + - uses: julia-actions/julia-runtest@v1 + env: + PYTHON: + - uses: julia-actions/julia-processcoverage@v1 + - uses: codecov/codecov-action@v1 + with: + file: lcov.info \ No newline at end of file diff --git a/.github/workflows/CompatHelper.yml b/.github/workflows/CompatHelper.yml new file mode 100644 index 0000000..9c96d0b --- /dev/null +++ b/.github/workflows/CompatHelper.yml @@ -0,0 +1,24 @@ +name: CompatHelper + +on: + schedule: + - cron: '00 00 * * *' + +jobs: + CompatHelper: + runs-on: ${{ matrix.os }} + strategy: + matrix: + julia-version: [1] + julia-arch: [x86] + os: [ubuntu-latest] + steps: + - uses: julia-actions/setup-julia@latest + with: + version: ${{ matrix.julia-version }} + - name: Pkg.add("CompatHelper") + run: julia -e 'using Pkg; Pkg.add("CompatHelper")' + - name: CompatHelper.main() + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: julia -e 'using CompatHelper; CompatHelper.main()' diff --git a/.github/workflows/TagBot.yml b/.github/workflows/TagBot.yml index c13ec18..623860f 100644 --- a/.github/workflows/TagBot.yml +++ b/.github/workflows/TagBot.yml @@ -1,12 +1,15 @@ name: TagBot on: - schedule: - - cron: 0 0 * * * + issue_comment: + types: + - created workflow_dispatch: jobs: TagBot: + if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot' runs-on: ubuntu-latest steps: - uses: JuliaRegistries/TagBot@v1 with: token: ${{ secrets.GITHUB_TOKEN }} + ssh: ${{ secrets.DOCUMENTER_KEY }} \ No newline at end of file diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 66c365a..0000000 --- a/.travis.yml +++ /dev/null @@ -1,30 +0,0 @@ -## Documentation: http://docs.travis-ci.com/user/languages/julia/ -language: julia -dist: xenial -os: - - linux - - osx -julia: - - 1 - - 1.3 - - nightly -notifications: - email: false -git: - depth: 99999999 - -jobs: - ## uncomment following lines to deploy documentation - # include: - # - stage: "Documentation" - # julia: 1.3 - # os: linux - # script: - # - julia --project=docs -e 'using Pkg; Pkg.instantiate(); Pkg.develop(PackageSpec(path=pwd()))' - # - julia --project=docs --color=yes docs/make.jl - # after_success: skip - allow_failures: - - julia: nightly -after_success: - - julia --project=test/coverage -e 'using Pkg; Pkg.instantiate()' - - julia --project=test/coverage test/coverage/coverage.jl diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index 5f55bea..0000000 --- a/appveyor.yml +++ /dev/null @@ -1,43 +0,0 @@ -environment: - matrix: - - julia_version: 1 - - julia_version: 1.3 - - julia_version: nightly - -platform: - - x86 # 32-bit - - x64 # 64-bit - -# # Uncomment the following lines to allow failures on nightly julia -# # (tests will run but not make your overall status red) -# matrix: -# allow_failures: -# - julia_version: nightly - -branches: - only: - - master - - /release-.*/ - -notifications: - - provider: Email - on_build_success: false - on_build_failure: false - on_build_status_changed: false - -install: - - ps: iex ((new-object net.webclient).DownloadString("https://raw.githubusercontent.com/JuliaCI/Appveyor.jl/version-1/bin/install.ps1")) - -build_script: - - echo "%JL_BUILD_SCRIPT%" - - C:\julia\bin\julia -e "%JL_BUILD_SCRIPT%" - -test_script: - - echo "%JL_TEST_SCRIPT%" - - C:\julia\bin\julia -e "%JL_TEST_SCRIPT%" - -# # Uncomment to support code coverage upload. Should only be enabled for packages -# # which would have coverage gaps without running on Windows -# on_success: -# - echo "%JL_CODECOV_SCRIPT%" -# - C:\julia\bin\julia -e "%JL_CODECOV_SCRIPT%"