From 65409605186ab0b579b9f512a556bae41ce7e4c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Riedemann?= <38795484+longemen3000@users.noreply.github.com> Date: Mon, 26 Jun 2023 17:17:18 -0400 Subject: [PATCH] 1.9 Extensions (#5) * transducers: 1.9 extensions * add OnlineStatsBase as test dependency * add explicit loading of OnlineStatsBase * update main enviroments * typo * explicitly import Transducers into extensions * import GenericInitialValue in OnlineStatsBaseExt * move OSNonZeroNObsError into Tranducers * typo in TransducersDataFramesExt * another typo in TransducersDataFramesExt * patch test to see if it the rest of the pkg works on nightly * tests: import Referenceables into scope * test on v1.9 * switch to regular Test.jl infrastructure * Delete vanilla-test.yml * Delete check-xfail.yml * Support 1.6+ * Support 1.6+ * revert changes to test infra * comment out test * revert commented out test * use Int instead of Int64 (32 bit tests) * trying to make test_ir pass * missing change in test_ir * skip doctests on 32 bit * skip doctests on 1.6 * fix test * Int64 -> Int * another fix to tests * bump version --------- Co-authored-by: Mason Protter --- .github/workflows/check-xfail.yml | 65 ------------------- .github/workflows/test.yml | 48 ++++---------- .github/workflows/vanilla-test.yml | 26 -------- Project.toml | 21 +++++- examples/tutorial_missings.jl | 12 ++-- .../TransducersBlockArraysExt.jl | 21 +++++- ext/TransducersDataFramesExt.jl | 15 +++++ .../TransducersLazyArraysExt.jl | 18 +++++ .../TransducersOnlineStatsBaseExt.jl | 54 +++++++++------ ext/TransducersReferenceablesExt.jl | 13 ++++ src/Transducers.jl | 34 ++++------ src/basics.jl | 2 +- src/interop/dataframes.jl | 3 - src/interop/referenceables.jl | 1 - test/Project.toml | 1 + test/__test_ir.jl | 13 ++-- test/environments/jl10/Project.toml | 1 + test/environments/main/Project.toml | 1 + test/environments/main_v2/Project.toml | 1 + test/environments/main_v3/Project.toml | 1 + test/test_doctest.jl | 21 ++++-- test/test_executor_types.jl | 4 +- test/test_onlinestats.jl | 1 + 23 files changed, 178 insertions(+), 199 deletions(-) delete mode 100644 .github/workflows/check-xfail.yml delete mode 100644 .github/workflows/vanilla-test.yml rename src/interop/blockarrays.jl => ext/TransducersBlockArraysExt.jl (75%) create mode 100644 ext/TransducersDataFramesExt.jl rename src/interop/lazyarrays.jl => ext/TransducersLazyArraysExt.jl (64%) rename src/interop/onlinestats.jl => ext/TransducersOnlineStatsBaseExt.jl (60%) create mode 100644 ext/TransducersReferenceablesExt.jl delete mode 100644 src/interop/dataframes.jl delete mode 100644 src/interop/referenceables.jl diff --git a/.github/workflows/check-xfail.yml b/.github/workflows/check-xfail.yml deleted file mode 100644 index 2783d0e2..00000000 --- a/.github/workflows/check-xfail.yml +++ /dev/null @@ -1,65 +0,0 @@ -name: Check xfail - -on: - pull_request: - -jobs: - xfail: - runs-on: ubuntu-latest - strategy: - matrix: - julia-version: ['^1'] - fail-fast: false - name: Test xfail Julia ${{ matrix.julia-version }} - steps: - - uses: actions/checkout@v2 - - name: Try to merge xfail branch - run: | - git_fetch_branch() { - refspec="refs/heads/$1:refs/remotes/origin/$1" - git fetch --unshallow origin "$refspec" || git fetch origin "$refspec" - } - set -ex - xfailbranch=xfail/"${GITHUB_HEAD_REF#refs/heads/}" - if git_fetch_branch "$xfailbranch" - then - git config --global user.email "$GITHUB_ACTOR@users.noreply.github.com" - git config --global user.name "$GITHUB_ACTOR" - git merge "origin/$xfailbranch" - else - echo "No branch named $xfailbranch" - fi - # Note: `$GITHUB_REF` contains something like "refs/pull/26/merge". - # It seems `$GITHUB_HEAD_REF` contains the correct branch name. - # See also GitHub context `github.head_ref`: - # https://help.github.com/en/actions/automating-your-workflow-with-github-actions/contexts-and-expression-syntax-for-github-actions#github-context - - - run: git log --graph --oneline -n10 - - run: git show --no-patch --format='format:%H %T' - - run: echo $GITHUB_SHA - - run: git diff $GITHUB_SHA - - - name: Check if xfail test is required - id: check-xfail-needed - run: | - xfailbranch=xfail/"${GITHUB_HEAD_REF#refs/heads/}" - if git rev-parse "refs/remotes/origin/$xfailbranch" -- > /dev/null - then - echo "::set-output name=need_xfail::yes" - else - echo "::set-output name=need_xfail::no" - fi - - - name: Setup julia - if: ${{ steps.check-xfail-needed.outputs.need_xfail == 'yes' }} - uses: julia-actions/setup-julia@v1 - with: - version: ${{ matrix.julia-version }} - - - run: julia -e 'using Pkg; pkg"add Run@0.1"' - if: ${{ steps.check-xfail-needed.outputs.need_xfail == 'yes' }} - - - run: julia -e 'using Run; Run.test(project = "test/environments/main", xfail = true)' - if: ${{ steps.check-xfail-needed.outputs.need_xfail == 'yes' }} - env: - JULIA_NUM_THREADS: '2' diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fe355d69..f7032011 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,50 +4,24 @@ on: push: branches: - master - tags: '*' pull_request: jobs: test: - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: matrix: - julia-version: - - '1.7' - - '1.6' - envname: ["main"] - include: - - julia-version: '1.0' - envname: "jl10" - - julia-version: '1.8' - envname: "main_v2" - - julia-version: 'nightly' - envname: "main_v3" - fail-fast: false - name: Test Julia ${{ matrix.julia-version }} + julia-version: ['1.6', '1.7', '1.8', '1.9', 'nightly'] + julia-arch: [x64, x86] + os: [ubuntu-latest] + steps: - uses: actions/checkout@v2 - - name: Setup julia - uses: julia-actions/setup-julia@v1 + - uses: julia-actions/setup-julia@v1 with: version: ${{ matrix.julia-version }} - - run: julia -e 'using Pkg; pkg"add Run@0.1"' - - run: julia -e 'using Run; Run.prepare("test/environments/${{ matrix.envname }}")' - - run: julia -e 'using Run; Run.test(project="test/environments/${{ matrix.envname }}")' - - uses: julia-actions/julia-processcoverage@v1 - - uses: codecov/codecov-action@v1 - with: - file: ./lcov.info - flags: Run.test - name: codecov-umbrella - - # A job that succeeds if and only if all jobs in `test` succeed. - all-success: - if: always() && github.event.pull_request - needs: test - runs-on: ubuntu-latest - steps: - # https://github.com/tkf/merge-conclusions-action - - uses: tkf/merge-conclusions-action@v1 - with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + arch: ${{ matrix.julia-arch }} + - uses: julia-actions/julia-buildpkg@v1 + - uses: julia-actions/julia-runtest@v1 + # with: + # annotate: true diff --git a/.github/workflows/vanilla-test.yml b/.github/workflows/vanilla-test.yml deleted file mode 100644 index 9e08bc04..00000000 --- a/.github/workflows/vanilla-test.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: Run test via Pkg.test() - -on: - push: - branches: - - master - pull_request: - -jobs: - vanilla-test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: julia-actions/setup-julia@v1 - with: - version: 1 - - uses: julia-actions/julia-buildpkg@latest - - uses: julia-actions/julia-runtest@latest - env: - JULIA_NUM_THREADS: "2" - - uses: julia-actions/julia-processcoverage@v1 - - uses: codecov/codecov-action@v1 - with: - file: ./lcov.info - flags: Pkg.test - name: codecov-umbrella diff --git a/Project.toml b/Project.toml index cb6f2e13..17f10274 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "Transducers" uuid = "28d57a85-8fef-5791-bfe6-a80928e7c999" authors = ["Takafumi Arakaki "] -version = "0.4.76" +version = "0.4.77" [deps] Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" @@ -33,7 +33,14 @@ Requires = "0.5, 1.0" Setfield = "0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 1" SplittablesBase = "0.1.2" Tables = "0.2, 1.0" -julia = "1" +julia = "1.6" + +[extensions] +TransducersBlockArraysExt = "BlockArrays" +TransducersDataFramesExt = "DataFrames" +TransducersLazyArraysExt = "LazyArrays" +TransducersOnlineStatsBaseExt = "OnlineStatsBase" +TransducersReferenceablesExt = "Referenceables" [extras] Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" @@ -51,6 +58,7 @@ LiterateTest = "d77d25b0-90d3-4a16-b10a-412a9d48f625" LoadAllPackages = "b37bcd2d-1570-475d-a8c6-9b4fae6d0ba9" Maybe = "334f122f-1118-46cc-837f-bff747ee6f78" OnlineStats = "a15396b6-48d5-5d58-9928-6d29437db91e" +OnlineStatsBase = "925886fa-5bf2-5e8e-b522-a9147a512338" PerformanceTestTools = "dc46b164-d16f-48ec-a853-60448fc869fe" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" @@ -63,4 +71,11 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" TypedTables = "9d95f2ec-7b3d-5a63-8d20-e2491e220bb9" [targets] -test = ["Aqua", "BlockArrays", "Compat", "DataFrames", "DataTools", "Dates", "Distributed", "Documenter", "Folds", "InteractiveUtils", "LazyArrays", "LiterateTest", "LoadAllPackages", "Maybe", "OnlineStats", "PerformanceTestTools", "Pkg", "Random", "Referenceables", "SparseArrays", "StaticArrays", "Statistics", "StructArrays", "Test", "TypedTables"] +test = ["Aqua", "BlockArrays", "Compat", "DataFrames", "DataTools", "Dates", "Distributed", "Documenter", "Folds", "InteractiveUtils", "LazyArrays", "LiterateTest", "LoadAllPackages", "Maybe", "OnlineStats", "OnlineStatsBase", "PerformanceTestTools", "Pkg", "Random", "Referenceables", "SparseArrays", "StaticArrays", "Statistics", "StructArrays", "Test", "TypedTables"] + +[weakdeps] +BlockArrays = "8e7c35d0-a365-5155-bbbb-fb81a777f24e" +DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" +LazyArrays = "5078a376-72f3-5289-bfd5-ec5146d43c02" +OnlineStatsBase = "925886fa-5bf2-5e8e-b522-a9147a512338" +Referenceables = "42d2dcc6-99eb-4e98-b66c-637b7d73030e" diff --git a/examples/tutorial_missings.jl b/examples/tutorial_missings.jl index c4618e0f..fb1569f9 100644 --- a/examples/tutorial_missings.jl +++ b/examples/tutorial_missings.jl @@ -247,16 +247,16 @@ using DataTools: rightif @test begin foldxl(rightif(<, last), filtered_pairs) -end === Pair{Int64,Union{Missing, Int}}(2, 3) +end === Pair{Int,Union{Missing, Int}}(2, 3) -# ### Side note: why `Pair{Int64,Union{Missing,Int}}`? +# ### Side note: why `Pair{Int,Union{Missing,Int}}`? # # The result type just above using `rightif` is -# `Pair{Int64,Union{Missing,Int}}`: +# `Pair{Int,Union{Missing,Int}}`: @test begin typeof(foldxl(rightif(<, last), filtered_pairs)) -end === Pair{Int64,Union{Missing,Int}} +end === Pair{Int,Union{Missing,Int}} # This is because that's the element type of `pairs([1, 3, missing, # 0])` and `rightif` does not re-construct the input `Pair` like @@ -264,7 +264,7 @@ end === Pair{Int64,Union{Missing,Int}} @test begin [1, 3, missing, 0] |> pairs |> first |> typeof -end === Pair{Int64,Union{Missing, Int}} +end === Pair{Int,Union{Missing, Int}} # We can avoid this by pre-processing the input with `MapSplat(Pair)`: @@ -359,7 +359,7 @@ end @test begin [1, 3, missing, 0] |> pairs |> - MapSplat(Pair) |> # avoid `Pair{Int64,Union{Missing, Int}}` + MapSplat(Pair) |> # avoid `Pair{Int,Union{Missing, Int}}` foldxl(TeeRF( Map(ismissing ∘ last)'(+), # count number of missings Filter(!(ismissing ∘ last))'(TeeRF( diff --git a/src/interop/blockarrays.jl b/ext/TransducersBlockArraysExt.jl similarity index 75% rename from src/interop/blockarrays.jl rename to ext/TransducersBlockArraysExt.jl index 9f0a8243..952acb93 100644 --- a/src/interop/blockarrays.jl +++ b/ext/TransducersBlockArraysExt.jl @@ -1,3 +1,15 @@ +module TransducersBlockArraysExt + +if isdefined(Base, :get_extension) + using Transducers: Transducers, @return_if_reduced, @next, @simd_if, complete + using Transducers.Setfield: @set + using BlockArrays +else + using ..Transducers: Transducers, @return_if_reduced, @next, @simd_if, complete + using ..Transducers.Setfield: @set + using ..BlockArrays +end + """ _foldl_blockarray(rf, acc, coll::BlockArrays.BlockArray) """ @@ -40,9 +52,7 @@ end ::Val{N₋₁}, ) where {RF,N₋₁} - blockaxes = BlockArrays.blockaxes - - @inbounds for b in blockaxes(coll, 1) + @inbounds for b in BlockArrays.blockaxes(coll, 1) array = coll[b, block...] @simd_if rf for k in axes(array, 1) acc = @next(rf, acc, array[k, offset...]) @@ -51,4 +61,9 @@ end return acc end +Transducers.__foldl__(rf, acc, coll::BlockArrays.BlockArray) = + _foldl_blockarray(rf, acc, coll) + # TODO: write reduce for BlockArrays which can be done in the "natural" order + +end diff --git a/ext/TransducersDataFramesExt.jl b/ext/TransducersDataFramesExt.jl new file mode 100644 index 00000000..ce5776d4 --- /dev/null +++ b/ext/TransducersDataFramesExt.jl @@ -0,0 +1,15 @@ +module TransducersDataFramesExt + +if isdefined(Base, :get_extension) + using Transducers + using DataFrames +else + using ..Transducers + using ..DataFrames +end + +Transducers.asfoldable(df::DataFrames.AbstractDataFrame) = DataFrames.eachrow(df) +# We can't use `Compat.eachrow` here. See: +# https://github.com/JuliaData/DataFrames.jl/pull/2067 + +end #module diff --git a/src/interop/lazyarrays.jl b/ext/TransducersLazyArraysExt.jl similarity index 64% rename from src/interop/lazyarrays.jl rename to ext/TransducersLazyArraysExt.jl index 5a7b4ceb..741c8e1d 100644 --- a/src/interop/lazyarrays.jl +++ b/ext/TransducersLazyArraysExt.jl @@ -1,3 +1,15 @@ +module TransducersLazyArraysExt + +if isdefined(Base, :get_extension) + using Transducers: Transducers, @return_if_reduced, @next, @simd_if, complete, foldlargs, foldl_nocomplete + using Transducers.Setfield: @set + using LazyArrays +else + using ..Transducers: Transducers, @return_if_reduced, @next, @simd_if, complete, foldlargs, foldl_nocomplete + using ..Transducers.Setfield: @set + using ..LazyArrays +end + @inline function _foldl_lazy_cat_vectors(rf, acc, vectors) isempty(vectors) && return complete(rf, acc) result = @return_if_reduced foldlargs(acc, vectors...) do acc, arr @@ -26,6 +38,12 @@ end end return complete(rf, acc) end + +Transducers.__foldl__(rf, acc, coll::LazyArrays.Hcat) = _foldl_lazy_hcat(rf, acc, coll) +Transducers.__foldl__(rf, acc, coll::LazyArrays.Vcat) = _foldl_lazy_vcat(rf, acc, coll) + # Vcat currently always is an `AbstractVector` or `AbstractMatrix` # TODO: write reduce for Vcat/Hcat which can be done in the "natural" order + +end diff --git a/src/interop/onlinestats.jl b/ext/TransducersOnlineStatsBaseExt.jl similarity index 60% rename from src/interop/onlinestats.jl rename to ext/TransducersOnlineStatsBaseExt.jl index 386394c5..7f9baa27 100644 --- a/src/interop/onlinestats.jl +++ b/ext/TransducersOnlineStatsBaseExt.jl @@ -1,3 +1,17 @@ +module TransducersOnlineStatsBaseExt + +if isdefined(Base, :get_extension) + using Transducers: Transducers, reducingfunction, Transducer, foldxl, foldxt, foldxd, extract_transducer, OSNonZeroNObsError + using Transducers.InitialValues + using InitialValues: GenericInitialValue + using OnlineStatsBase +else + using ..Transducers: Transducers, reducingfunction, Transducer, foldxl, foldxt, foldxd, extract_transducer, OSNonZeroNObsError + using ..Transducers.InitialValues + using ..InitialValues: GenericInitialValue + using ..OnlineStatsBase +end + """ Transducer(o::OnlineStat) @@ -22,8 +36,8 @@ julia> collect(Transducer(Mean()), 1:4) 2.5 ``` """ -Transducer(o::OnlineStatsBase.OnlineStat) = - Map(OnlineStatsBase.value) ∘ Scan(OnlineStatsBase.fit!, CopyInit(o)) +Transducers.Transducer(o::OnlineStatsBase.OnlineStat) = +Transducers.Map(OnlineStatsBase.value) ∘ Transducers.Scan(OnlineStatsBase.fit!, Transducers.CopyInit(o)) # TODO: implement `combine` """ @@ -58,17 +72,18 @@ Mean: n=4 | value=7.5 [`foldxd`](@ref) can be used instead of `foldxt`. However the usual caveats of code availability for Distributed.jl apply. """ -reducingfunction(xf::Transducer, stat::OnlineStatsBase.OnlineStat; kwargs...) = - reducingfunction(xf, reducingfunction(stat); kwargs...) -reducingfunction(stat::OnlineStatsBase.OnlineStat) = OnlineStatReducingFunction(stat) +Transducers.reducingfunction(xf::Transducer, stat::OnlineStatsBase.OnlineStat; kwargs...) = + Transducers.reducingfunction(xf, reducingfunction(stat); kwargs...) -@inline _reducingfunction(xf, stat::OnlineStatsBase.OnlineStat; kwargs...) = - _reducingfunction(xf, reducingfunction(stat); kwargs...) +@inline Transducers._reducingfunction(xf, stat::OnlineStatsBase.OnlineStat; kwargs...) = +Transducers._reducingfunction(xf, reducingfunction(stat); kwargs...) struct OnlineStatReducingFunction{T<:OnlineStatsBase.OnlineStat} <: Function init::T end +Transducers.reducingfunction(stat::OnlineStatsBase.OnlineStat) = OnlineStatReducingFunction(stat) + InitialValues.hasinitialvalue(::Type{<:OnlineStatReducingFunction}) = true # `GenericInitialValue{T}` for case when `init=Init(op::OnlineStat)` @@ -84,25 +99,22 @@ function (::OnlineStatReducingFunction)(acc, x) return acc end -combine(::OnlineStatReducingFunction{T}, a::T, b::T) where {T<:OnlineStatsBase.OnlineStat} = +Transducers.combine(::OnlineStatReducingFunction{T}, a::T, b::T) where {T<:OnlineStatsBase.OnlineStat} = merge!(a, b) -combine( + Transducers.combine( ::OnlineStatReducingFunction{T}, a::Union{T,OSInit{T}}, ::OSInit{T}, ) where {T<:OnlineStatsBase.OnlineStat} = a -combine( +Transducers.combine( ::OnlineStatReducingFunction{T}, ::OSInit{T}, b::T, ) where {T<:OnlineStatsBase.OnlineStat} = b -const OSNonZeroNObsError = ArgumentError( - "An `OnlineStat` with one or more observations cannot be used with " * - "`foldxt` and `foldxd`.", -) + function validate_reduce_ostat(stat) if OnlineStatsBase.nobs(stat) != 0 @@ -112,26 +124,28 @@ function validate_reduce_ostat(stat) end # Method plumbing: -foldxl(stat::OnlineStatsBase.OnlineStat, xform::Transducer, itr; kwargs...) = +Transducers.foldxl(stat::OnlineStatsBase.OnlineStat, xform::Transducer, itr; kwargs...) = foldxl(reducingfunction(stat), xform, itr; kwargs...) -foldxl(stat::OnlineStatsBase.OnlineStat, foldable; kwargs...) = +Transducers.foldxl(stat::OnlineStatsBase.OnlineStat, foldable; kwargs...) = foldxl(reducingfunction(stat), extract_transducer(foldable)...; kwargs...) -foldxt(stat::OnlineStatsBase.OnlineStat, xform::Transducer, itr; kwargs...) = +Transducers.foldxt(stat::OnlineStatsBase.OnlineStat, xform::Transducer, itr; kwargs...) = foldxt(reducingfunction(validate_reduce_ostat(stat)), xform, itr; kwargs...) -foldxt(stat::OnlineStatsBase.OnlineStat, foldable; kwargs...) = foldxt( +Transducers.foldxt(stat::OnlineStatsBase.OnlineStat, foldable; kwargs...) = foldxt( reducingfunction(validate_reduce_ostat(stat)), extract_transducer(foldable)...; kwargs..., ) -foldxd(stat::OnlineStatsBase.OnlineStat, xform::Transducer, itr; kwargs...) = +Transducers.foldxd(stat::OnlineStatsBase.OnlineStat, xform::Transducer, itr; kwargs...) = foldxd(reducingfunction(validate_reduce_ostat(stat)), xform, itr; kwargs...) -foldxd(stat::OnlineStatsBase.OnlineStat, foldable; kwargs...) = foldxd( +Transducers.foldxd(stat::OnlineStatsBase.OnlineStat, foldable; kwargs...) = foldxd( reducingfunction(validate_reduce_ostat(stat)), extract_transducer(foldable)...; kwargs..., ) + +end #module diff --git a/ext/TransducersReferenceablesExt.jl b/ext/TransducersReferenceablesExt.jl new file mode 100644 index 00000000..6cd9aa7f --- /dev/null +++ b/ext/TransducersReferenceablesExt.jl @@ -0,0 +1,13 @@ +module TransducersReferenceablesExt + +if isdefined(Base, :get_extension) + using Transducers + using Referenceables +else + using ..Transducers + using ..Referenceables +end + +@inline Transducers.executor_type(x::Referenceables.Referenceable) = Transducers.executor_type(parent(x)) + +end #module diff --git a/src/Transducers.jl b/src/Transducers.jl index 6aa599a8..9368e215 100644 --- a/src/Transducers.jl +++ b/src/Transducers.jl @@ -170,28 +170,20 @@ include("comprehensions.jl") include("progress.jl") include("deprecated.jl") -include("interop/blockarrays.jl") -include("interop/lazyarrays.jl") +#used by TransducersOnlineStatsBaseExt, but exported directly in tests +const OSNonZeroNObsError = ArgumentError( + "An `OnlineStat` with one or more observations cannot be used with " * + "`foldxt` and `foldxd`.", +) -function __init__() - @require BlockArrays="8e7c35d0-a365-5155-bbbb-fb81a777f24e" begin - __foldl__(rf, acc, coll::BlockArrays.BlockArray) = - _foldl_blockarray(rf, acc, coll) - end - @require LazyArrays="5078a376-72f3-5289-bfd5-ec5146d43c02" begin - __foldl__(rf, acc, coll::LazyArrays.Hcat) = - _foldl_lazy_hcat(rf, acc, coll) - __foldl__(rf, acc, coll::LazyArrays.Vcat) = - _foldl_lazy_vcat(rf, acc, coll) - end - @require OnlineStatsBase="925886fa-5bf2-5e8e-b522-a9147a512338" begin - include("interop/onlinestats.jl") - end - @require DataFrames="a93c6f00-e57d-5684-b7b6-d8193f3e46c0" begin - include("interop/dataframes.jl") - end - @require Referenceables="42d2dcc6-99eb-4e98-b66c-637b7d73030e" begin - include("interop/referenceables.jl") +if !isdefined(Base,:get_extension) + using Requires + function __init__() + @require BlockArrays="8e7c35d0-a365-5155-bbbb-fb81a777f24e" include("../ext/TransducersBlockArraysExt.jl") + @require LazyArrays="5078a376-72f3-5289-bfd5-ec5146d43c02" include("../ext/TransducersLazyArraysExt.jl") + @require DataFrames="a93c6f00-e57d-5684-b7b6-d8193f3e46c0" include("../ext/TransducersDataFramesExt.jl") + @require OnlineStatsBase="925886fa-5bf2-5e8e-b522-a9147a512338" include("../ext/TransducersOnlineStatsBaseExt.jl") + @require Referenceables="42d2dcc6-99eb-4e98-b66c-637b7d73030e" include("../ext/TransducersReferenceablesExt.jl") end end diff --git a/src/basics.jl b/src/basics.jl index 4bd8db18..5aa1ce4a 100644 --- a/src/basics.jl +++ b/src/basics.jl @@ -54,7 +54,7 @@ prefixed_type_name(@nospecialize x) = # https://github.com/JuliaLang/julia/pull/29466 const DenseSubVector{T} = - SubArray{T, 1, Vector{T}, Tuple{UnitRange{Int64}}, true} + SubArray{T, 1, Vector{T}, Tuple{UnitRange{Int}}, true} # https://github.com/JuliaLang/julia/pull/33533 if VERSION < v"1.4" diff --git a/src/interop/dataframes.jl b/src/interop/dataframes.jl deleted file mode 100644 index 3348a491..00000000 --- a/src/interop/dataframes.jl +++ /dev/null @@ -1,3 +0,0 @@ -asfoldable(df::DataFrames.AbstractDataFrame) = DataFrames.eachrow(df) -# We can't use `Compat.eachrow` here. See: -# https://github.com/JuliaData/DataFrames.jl/pull/2067 diff --git a/src/interop/referenceables.jl b/src/interop/referenceables.jl deleted file mode 100644 index e78fb884..00000000 --- a/src/interop/referenceables.jl +++ /dev/null @@ -1 +0,0 @@ -@inline executor_type(x::Referenceables.Referenceable) = executor_type(parent(x)) diff --git a/test/Project.toml b/test/Project.toml index 1938057b..a875bcab 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -21,6 +21,7 @@ Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" Maybe = "334f122f-1118-46cc-837f-bff747ee6f78" MicroCollections = "128add7d-3638-4c79-886c-908ea0c25c34" OnlineStats = "a15396b6-48d5-5d58-9928-6d29437db91e" +OnlineStatsBase = "925886fa-5bf2-5e8e-b522-a9147a512338" PerformanceTestTools = "dc46b164-d16f-48ec-a853-60448fc869fe" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" diff --git a/test/__test_ir.jl b/test/__test_ir.jl index 21077424..0ab5a382 100644 --- a/test/__test_ir.jl +++ b/test/__test_ir.jl @@ -24,7 +24,8 @@ matchedlines(r, s) = [m.match for m in eachmatch(r, s)] anyunions(s) = matchedlines(r".*UNION.*", s) nmatches(r, s) = count(_ -> true, eachmatch(r, s)) - +const __is32bit = Int == Int32 +const __width_ir_fmul = __is32bit ? 2 : 4 @testset "map!" begin xf = opcompose(Filter(x -> -0.5 < x < 0.5), Map(x -> 2x)) xs = Float64[] @@ -35,16 +36,16 @@ nmatches(r, s) = count(_ -> true, eachmatch(r, s)) # compiler becomes _extremely_ smart). ir = llvm_ir(map!, (xf, ys, xs)) @debug "map!/history" LLVM_IR=Text(ir) - @test_broken nmatches(r"fmul <[0-9]+ x double>", ir) >= 4 - @test_broken nmatches(r"fcmp [a-z]* <[0-9]+ x double>", ir) >= 4 + @test_broken nmatches(r"fmul <[0-9]+ x double>", ir) >= __width_ir_fmul + @test_broken nmatches(r"fcmp [a-z]* <[0-9]+ x double>", ir) >= __width_ir_fmul end @testset for simd in [false, true, :ivdep] args = _prepare_map(xf, ys, xs, simd) ir = llvm_ir(_map!, args) @debug "map!/simd=$simd" LLVM_IR=Text(ir) - @test nmatches(r"fmul <[0-9]+ x double>", ir) >= 4 - @test nmatches(r"fcmp [a-z]* <[0-9]+ x double>", ir) >= 4 + @test nmatches(r"fmul <[0-9]+ x double>", ir) >= __width_ir_fmul + @test nmatches(r"fcmp [a-z]* <[0-9]+ x double>", ir) >= __width_ir_fmul end end @@ -94,7 +95,7 @@ unsafe_setter(ys) = ir = llvm_ir(transduce, (rf, nothing, xs)) @debug "foreach SIMD/$key" LLVM_IR=Text(ir) - @test nmatches(r"fmul <[0-9]+ x double>", ir) >= 4 + @test nmatches(r"fmul <[0-9]+ x double>", ir) >= __width_ir_fmul end end diff --git a/test/environments/jl10/Project.toml b/test/environments/jl10/Project.toml index 3ed0da07..c9b1b95e 100644 --- a/test/environments/jl10/Project.toml +++ b/test/environments/jl10/Project.toml @@ -20,6 +20,7 @@ Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" Maybe = "334f122f-1118-46cc-837f-bff747ee6f78" MicroCollections = "128add7d-3638-4c79-886c-908ea0c25c34" OnlineStats = "a15396b6-48d5-5d58-9928-6d29437db91e" +OnlineStatsBase = "925886fa-5bf2-5e8e-b522-a9147a512338" PerformanceTestTools = "dc46b164-d16f-48ec-a853-60448fc869fe" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" diff --git a/test/environments/main/Project.toml b/test/environments/main/Project.toml index 1938057b..a875bcab 100644 --- a/test/environments/main/Project.toml +++ b/test/environments/main/Project.toml @@ -21,6 +21,7 @@ Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" Maybe = "334f122f-1118-46cc-837f-bff747ee6f78" MicroCollections = "128add7d-3638-4c79-886c-908ea0c25c34" OnlineStats = "a15396b6-48d5-5d58-9928-6d29437db91e" +OnlineStatsBase = "925886fa-5bf2-5e8e-b522-a9147a512338" PerformanceTestTools = "dc46b164-d16f-48ec-a853-60448fc869fe" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" diff --git a/test/environments/main_v2/Project.toml b/test/environments/main_v2/Project.toml index 1938057b..a875bcab 100644 --- a/test/environments/main_v2/Project.toml +++ b/test/environments/main_v2/Project.toml @@ -21,6 +21,7 @@ Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" Maybe = "334f122f-1118-46cc-837f-bff747ee6f78" MicroCollections = "128add7d-3638-4c79-886c-908ea0c25c34" OnlineStats = "a15396b6-48d5-5d58-9928-6d29437db91e" +OnlineStatsBase = "925886fa-5bf2-5e8e-b522-a9147a512338" PerformanceTestTools = "dc46b164-d16f-48ec-a853-60448fc869fe" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" diff --git a/test/environments/main_v3/Project.toml b/test/environments/main_v3/Project.toml index 1938057b..a875bcab 100644 --- a/test/environments/main_v3/Project.toml +++ b/test/environments/main_v3/Project.toml @@ -21,6 +21,7 @@ Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" Maybe = "334f122f-1118-46cc-837f-bff747ee6f78" MicroCollections = "128add7d-3638-4c79-886c-908ea0c25c34" OnlineStats = "a15396b6-48d5-5d58-9928-6d29437db91e" +OnlineStatsBase = "925886fa-5bf2-5e8e-b522-a9147a512338" PerformanceTestTools = "dc46b164-d16f-48ec-a853-60448fc869fe" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" diff --git a/test/test_doctest.jl b/test/test_doctest.jl index fb2e2a5e..05f45647 100644 --- a/test/test_doctest.jl +++ b/test/test_doctest.jl @@ -7,12 +7,23 @@ using Transducers # Workaround: Failed to evaluate `CurrentModule = Transducers` in `@meta` block. @eval Main import Transducers -@testset "/docs" begin - doctest(Transducers; manual=true) -end +const __is32bit = Int == Int32 + + +if !__is32bit && Base.VERSION >= v"1.7" + #the docs are meant with 64 bits in mind. + #so we skip the doctests on 32 bits,because of the Int issue. + + #the version issue is https://github.com/joshday/OnlineStatsBase.jl/issues/32 . + #should be solved by https://github.com/joshday/OnlineStatsBase.jl/pull/34 . + #in the meanwhile, skip doctests for 1.6 + @testset "/docs" begin + doctest(Transducers; manual=true) + end -@testset "/test/doctests" begin - doctest(joinpath((@__DIR__), "doctests"), Module[]) + @testset "/test/doctests" begin + doctest(joinpath((@__DIR__), "doctests"), Module[]) + end end end # module diff --git a/test/test_executor_types.jl b/test/test_executor_types.jl index 341e84aa..f78efdf4 100644 --- a/test/test_executor_types.jl +++ b/test/test_executor_types.jl @@ -1,7 +1,7 @@ module TestExecutorTypes include("preamble.jl") -using Referenceables: referenceable +using Referenceables: Referenceables, referenceable using Transducers: Executor, PreferParallel, executor_type, maybe_set_simd struct Opinionated{T<:Executor} <: AbstractVector{Int} end @@ -30,7 +30,7 @@ Base.size(::Opinionated) = (0,) (referenceable([]), PreferParallel), (referenceable(Opinionated(SequentialEx)), SequentialEx), ] - @test @inferred(executor_type(xs)) === ex + @test @inferred(executor_type(xs)) === ex end end diff --git a/test/test_onlinestats.jl b/test/test_onlinestats.jl index 2341f439..3d8d3dc5 100644 --- a/test/test_onlinestats.jl +++ b/test/test_onlinestats.jl @@ -1,6 +1,7 @@ module TestOnlineStats include("preamble.jl") +using OnlineStatsBase #required to pass a test using OnlineStats: Mean, EqualWeight using Transducers: InitialValue, combine, InitialValue, reducingfunction