From 91dd60ba685a22f7fa05c26d2314a1962f165493 Mon Sep 17 00:00:00 2001 From: Maarten Pronk Date: Fri, 15 Dec 2023 18:13:09 -0800 Subject: [PATCH 1/2] Use search-after header for scrolling instead of pagination. --- src/search.jl | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/search.jl b/src/search.jl index e70e1c6..d89ee90 100644 --- a/src/search.jl +++ b/src/search.jl @@ -208,12 +208,11 @@ function earthdata_search(; after::Union{Nothing,DateTime} = nothing, all_pages::Bool = true, page_size = 2000, - page_num = 1, + page_num = 1, # unused umm = false, verbose = 0, ) q = Dict( - "page_num" => page_num, "page_size" => page_size, "short_name" => short_name, ) @@ -234,10 +233,10 @@ function earthdata_search(; cgranules = parsef(r) granules = Vector{NamedTuple}() append!(granules, cgranules) - while (length(cgranules) == page_size) && all_pages + while !(length(cgranules) < page_size) && haskey(Dict(r.headers), "CMR-Search-After") && all_pages @warn "Found more than $page_size granules, requesting another $page_size..." - q["page_num"] += 1 - r = HTTP.get(qurl, query = q, verbose = verbose, status_exception = false) + v = get(Dict(r.headers), SubString("CMR-Search-After"), "") + r = HTTP.get(qurl, query = q, ["CMR-Search-After" => v], verbose = verbose, status_exception = false) HTTP.iserror(r) && error(parse_cmr_error(r)) cgranules = parsef(r) append!(granules, cgranules) From 36c72f54ecc9d25a2322498a87a95f1f5e509a39 Mon Sep 17 00:00:00 2001 From: Maarten Pronk Date: Sat, 16 Dec 2023 09:11:01 -0800 Subject: [PATCH 2/2] Fix dynamic test. --- test/runtests.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/runtests.jl b/test/runtests.jl index 714b678..3518803 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -82,7 +82,7 @@ empty_bbox = (min_x = 4.0, min_y = 40.0, max_x = 5.0, max_y = 50.0) # Time @test length(SpaceLiDAR.search(:ICESat2, :ATL08, after = DateTime(2019, 12, 12), before = DateTime(2019, 12, 13))) == 161 - @test length(SpaceLiDAR.search(:ICESat2, :ATL08, before = now() - Year(5))) == 0 + @test length(SpaceLiDAR.search(:ICESat2, :ATL08, before = DateTime(2017, 12, 12))) == 0 @test length(SpaceLiDAR.search(:ICESat2, :ATL08, after = now())) == 0 @test_throws ErrorException SpaceLiDAR.search(:ICESat2, :ATL08, after = now() - Month(47), before = now() - Month(48)) end