From 31c2cf3dc21d3861637e06e7bd2c2893f399969d Mon Sep 17 00:00:00 2001 From: ST John Date: Fri, 24 Sep 2021 16:59:46 +0300 Subject: [PATCH 01/29] add Laplace approximation --- LICENSE | 2 +- Project.toml | 2 + examples/a-regression/Manifest.toml | 84 +- examples/b-classification/Manifest.toml | 110 +- examples/c-comparisons/Manifest.toml | 1256 +++++++++++++++++++++++ examples/c-comparisons/Project.toml | 11 + examples/c-comparisons/script.jl | 125 +++ src/ApproximateGPs.jl | 9 + src/laplace.jl | 332 ++++++ test/Project.toml | 12 + test/laplace.jl | 182 ++++ test/runtests.jl | 37 +- 12 files changed, 2057 insertions(+), 105 deletions(-) create mode 100644 examples/c-comparisons/Manifest.toml create mode 100644 examples/c-comparisons/Project.toml create mode 100644 examples/c-comparisons/script.jl create mode 100644 src/laplace.jl create mode 100644 test/laplace.jl diff --git a/LICENSE b/LICENSE index aa3e25a2..dde3dc88 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ Copyright (c) 2021 -Ross Viljoen +The JuliaGaussianProcess Contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/Project.toml b/Project.toml index 3a9842ab..6028e8f7 100644 --- a/Project.toml +++ b/Project.toml @@ -9,6 +9,7 @@ ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" FastGaussQuadrature = "442a2c76-b920-505d-bb47-c5924d526838" FillArrays = "1a297f60-69ca-5386-bcde-b61e274b549b" +ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" GPLikelihoods = "6031954c-0455-49d7-b3b9-3e1c99afaf40" KLDivergences = "3c9cd921-3d3f-41e2-830c-e020174918cc" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" @@ -23,6 +24,7 @@ ChainRulesCore = "1" Distributions = "0.25" FastGaussQuadrature = "0.4" FillArrays = "0.12" +ForwardDiff = "0.10" GPLikelihoods = "0.1, 0.2" KLDivergences = "0.2.1" Reexport = "1" diff --git a/examples/a-regression/Manifest.toml b/examples/a-regression/Manifest.toml index 913565cb..bf1717ca 100644 --- a/examples/a-regression/Manifest.toml +++ b/examples/a-regression/Manifest.toml @@ -24,19 +24,19 @@ uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" version = "3.3.1" [[ApproximateGPs]] -deps = ["AbstractGPs", "ChainRulesCore", "Distributions", "FastGaussQuadrature", "FillArrays", "GPLikelihoods", "KLDivergences", "LinearAlgebra", "Reexport", "SpecialFunctions", "Statistics", "StatsBase"] +deps = ["AbstractGPs", "ChainRulesCore", "Distributions", "FastGaussQuadrature", "FillArrays", "ForwardDiff", "GPLikelihoods", "KLDivergences", "LinearAlgebra", "QuadGK", "Reexport", "SpecialFunctions", "Statistics", "StatsBase"] path = "../.." uuid = "298c2ebc-0411-48ad-af38-99e88101b606" -version = "0.1.0" +version = "0.1.1" [[ArgTools]] uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" [[ArrayInterface]] -deps = ["IfElse", "LinearAlgebra", "Requires", "SparseArrays", "Static"] -git-tree-sha1 = "d84c956c4c0548b4caf0e4e96cf5b6494b5b1529" +deps = ["Compat", "IfElse", "LinearAlgebra", "Requires", "SparseArrays", "Static"] +git-tree-sha1 = "b8d49c34c3da35f220e7295659cd0bab8e739fed" uuid = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9" -version = "3.1.32" +version = "3.1.33" [[Artifacts]] uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" @@ -81,9 +81,9 @@ version = "1.11.5" [[ChainRulesCore]] deps = ["Compat", "LinearAlgebra", "SparseArrays"] -git-tree-sha1 = "4ce9393e871aca86cc457d9f66976c3da6902ea7" +git-tree-sha1 = "bd4afa1fdeec0c8b89dad3c6e92bc6e3b0fec9ce" uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" -version = "1.4.0" +version = "1.6.0" [[CodecZlib]] deps = ["TranscodingStreams", "Zlib_jll"] @@ -117,9 +117,9 @@ version = "0.3.0" [[Compat]] deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "SHA", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"] -git-tree-sha1 = "4866e381721b30fac8dda4c8cb1d9db45c8d2994" +git-tree-sha1 = "1a90210acd935f222ea19657f143004d2c2a1117" uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" -version = "3.37.0" +version = "3.38.0" [[CompilerSupportLibraries_jll]] deps = ["Artifacts", "Libdl"] @@ -137,9 +137,9 @@ uuid = "d38c429a-6771-53c6-b99e-75d170b6e991" version = "0.5.7" [[DataAPI]] -git-tree-sha1 = "bec2532f8adb82005476c141ec23e921fc20971b" +git-tree-sha1 = "cc70b17275652eb47bc9e5f81635981f13cea5c8" uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" -version = "1.8.0" +version = "1.9.0" [[DataStructures]] deps = ["Compat", "InteractiveUtils", "OrderedCollections"] @@ -168,9 +168,9 @@ version = "1.0.3" [[DiffRules]] deps = ["NaNMath", "Random", "SpecialFunctions"] -git-tree-sha1 = "3ed8fa7178a10d1cd0f1ca524f249ba6937490c0" +git-tree-sha1 = "7220bc21c33e990c14f4a9a319b1d242ebc5b269" uuid = "b552c78f-8df3-52c6-915a-8e097449b14b" -version = "1.3.0" +version = "1.3.1" [[Distances]] deps = ["LinearAlgebra", "Statistics", "StatsAPI"] @@ -235,9 +235,9 @@ version = "0.4.7" [[FillArrays]] deps = ["LinearAlgebra", "Random", "SparseArrays", "Statistics"] -git-tree-sha1 = "caf289224e622f518c9dbfe832cdafa17d7c80a6" +git-tree-sha1 = "7f6ad1a7f4621b4ab8e554133dade99ebc6e7221" uuid = "1a297f60-69ca-5386-bcde-b61e274b549b" -version = "0.12.4" +version = "0.12.5" [[FixedPointNumbers]] deps = ["Statistics"] @@ -300,9 +300,9 @@ version = "0.2.0" [[GPUArrays]] deps = ["Adapt", "LinearAlgebra", "Printf", "Random", "Serialization", "Statistics"] -git-tree-sha1 = "8fac1cf7d6ce0f2249c7acaf25d22e1e85c4a07f" +git-tree-sha1 = "7c39d767a9c55fafd01f7bc8b3fd0adf175fbc97" uuid = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7" -version = "8.0.2" +version = "8.1.0" [[GPUCompiler]] deps = ["ExprTools", "InteractiveUtils", "LLVM", "Libdl", "Logging", "TimerOutputs", "UUIDs"] @@ -312,9 +312,9 @@ version = "0.12.9" [[GR]] deps = ["Base64", "DelimitedFiles", "GR_jll", "HTTP", "JSON", "Libdl", "LinearAlgebra", "Pkg", "Printf", "Random", "Serialization", "Sockets", "Test", "UUIDs"] -git-tree-sha1 = "182da592436e287758ded5be6e32c406de3a2e47" +git-tree-sha1 = "c2178cfbc0a5a552e16d097fae508f2024de61a3" uuid = "28b8d3ca-fb5f-59d9-8090-bfdbd6d07a71" -version = "0.58.1" +version = "0.59.0" [[GR_jll]] deps = ["Artifacts", "Bzip2_jll", "Cairo_jll", "FFMPEG_jll", "Fontconfig_jll", "GLFW_jll", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Libtiff_jll", "Pixman_jll", "Pkg", "Qt5Base_jll", "Zlib_jll", "libpng_jll"] @@ -449,9 +449,9 @@ version = "3.100.1+0" [[LLVM]] deps = ["CEnum", "LLVMExtra_jll", "Libdl", "Printf", "Unicode"] -git-tree-sha1 = "8fb1a675d1b51885a78bc980fbf1944279880f97" +git-tree-sha1 = "36d95ecdfbc3240d728f68d73064d5b097fbf2ef" uuid = "929cbde3-209d-540e-8aea-75f648917ca0" -version = "4.5.1" +version = "4.5.2" [[LLVMExtra_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] @@ -635,6 +635,10 @@ git-tree-sha1 = "7937eda4681660b4d6aeeecc2f7e1c81c8ee4e2f" uuid = "e7412a2a-1a6e-54c0-be00-318e2571c051" version = "1.3.5+0" +[[OpenLibm_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "05823500-19ac-5b8b-9628-191a04bc5112" + [[OpenSSL_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] git-tree-sha1 = "15003dcb7d8db3c6c857fda14891a539a8f2705a" @@ -672,9 +676,9 @@ version = "0.11.1" [[Parsers]] deps = ["Dates"] -git-tree-sha1 = "438d35d2d95ae2c5e8780b330592b6de8494e779" +git-tree-sha1 = "9d8c00ef7a8d110787ff6f170579846f776133a9" uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" -version = "2.0.3" +version = "2.0.4" [[Pixman_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] @@ -700,9 +704,9 @@ version = "1.0.14" [[Plots]] deps = ["Base64", "Contour", "Dates", "Downloads", "FFMPEG", "FixedPointNumbers", "GR", "GeometryBasics", "JSON", "Latexify", "LinearAlgebra", "Measures", "NaNMath", "PlotThemes", "PlotUtils", "Printf", "REPL", "Random", "RecipesBase", "RecipesPipeline", "Reexport", "Requires", "Scratch", "Showoff", "SparseArrays", "Statistics", "StatsBase", "UUIDs"] -git-tree-sha1 = "2dbafeadadcf7dadff20cd60046bba416b4912be" +git-tree-sha1 = "457b13497a3ea4deb33d273a6a5ea15c25c0ebd9" uuid = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" -version = "1.21.3" +version = "1.22.2" [[Preferences]] deps = ["TOML"] @@ -726,9 +730,9 @@ version = "5.15.3+0" [[QuadGK]] deps = ["DataStructures", "LinearAlgebra"] -git-tree-sha1 = "12fbe86da16df6679be7521dfb39fbc861e1dc7b" +git-tree-sha1 = "78aadffb3efd2155af139781b8a8df1ef279ea39" uuid = "1fd47b50-473d-5c70-9696-f719f8f3bcdc" -version = "2.4.1" +version = "2.4.2" [[REPL]] deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] @@ -757,9 +761,9 @@ version = "1.1.2" [[RecipesPipeline]] deps = ["Dates", "NaNMath", "PlotUtils", "RecipesBase"] -git-tree-sha1 = "d4491becdc53580c6dadb0f6249f90caae888554" +git-tree-sha1 = "7ad0dfa8d03b7bcf8c597f59f5292801730c55b8" uuid = "01d81517-befc-4cb6-b9ec-a95719d0359c" -version = "0.4.0" +version = "0.4.1" [[Reexport]] git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" @@ -820,10 +824,10 @@ deps = ["LinearAlgebra", "Random"] uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" [[SpecialFunctions]] -deps = ["ChainRulesCore", "LogExpFunctions", "OpenSpecFun_jll"] -git-tree-sha1 = "a322a9493e49c5f3a10b50df3aedaf1cdb3244b7" +deps = ["ChainRulesCore", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"] +git-tree-sha1 = "ad42c30a6204c74d264692e633133dcea0e8b14e" uuid = "276daf66-3868-5448-9aa4-cd146d93841b" -version = "1.6.1" +version = "1.6.2" [[Static]] deps = ["IfElse"] @@ -860,9 +864,9 @@ version = "0.9.10" [[StructArrays]] deps = ["Adapt", "DataAPI", "StaticArrays", "Tables"] -git-tree-sha1 = "f41020e84127781af49fc12b7e92becd7f5dd0ba" +git-tree-sha1 = "2ce41e0d042c60ecd131e9fb7154a3bfadbf50d3" uuid = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" -version = "0.6.2" +version = "0.6.3" [[SuiteSparse]] deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"] @@ -900,9 +904,9 @@ uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [[TimerOutputs]] deps = ["ExprTools", "Printf"] -git-tree-sha1 = "209a8326c4f955e2442c07b56029e88bb48299c7" +git-tree-sha1 = "7cb456f358e8f9d102a8b25e8dfedf58fa5689bc" uuid = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f" -version = "0.5.12" +version = "0.5.13" [[TranscodingStreams]] deps = ["Random", "Test"] @@ -1074,9 +1078,9 @@ version = "1.4.0+3" [[ZipFile]] deps = ["Libdl", "Printf", "Zlib_jll"] -git-tree-sha1 = "c3a5637e27e914a7a445b8d0ad063d701931e9f7" +git-tree-sha1 = "3593e69e469d2111389a9bd06bac1f3d730ac6de" uuid = "a5390f91-8eb1-5f08-bee0-b1d1ffed6cea" -version = "0.9.3" +version = "0.9.4" [[Zlib_jll]] deps = ["Libdl"] @@ -1090,9 +1094,9 @@ version = "1.5.0+0" [[Zygote]] deps = ["AbstractFFTs", "ChainRules", "ChainRulesCore", "DiffRules", "Distributed", "FillArrays", "ForwardDiff", "IRTools", "InteractiveUtils", "LinearAlgebra", "MacroTools", "NaNMath", "Random", "Requires", "SpecialFunctions", "Statistics", "ZygoteRules"] -git-tree-sha1 = "ffbf36ba9cd8476347486a013c93590b910a4855" +git-tree-sha1 = "4b799addc63aa77ad4112cede8086564d9068511" uuid = "e88e6eb3-aa80-5325-afca-941959d7151f" -version = "0.6.21" +version = "0.6.22" [[ZygoteRules]] deps = ["MacroTools"] diff --git a/examples/b-classification/Manifest.toml b/examples/b-classification/Manifest.toml index 6adb84dd..c596c51e 100644 --- a/examples/b-classification/Manifest.toml +++ b/examples/b-classification/Manifest.toml @@ -19,10 +19,10 @@ uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" version = "3.3.1" [[ApproximateGPs]] -deps = ["AbstractGPs", "ChainRulesCore", "Distributions", "FastGaussQuadrature", "FillArrays", "GPLikelihoods", "KLDivergences", "LinearAlgebra", "Reexport", "SpecialFunctions", "Statistics", "StatsBase"] +deps = ["AbstractGPs", "ChainRulesCore", "Distributions", "FastGaussQuadrature", "FillArrays", "ForwardDiff", "GPLikelihoods", "KLDivergences", "LinearAlgebra", "QuadGK", "Reexport", "SpecialFunctions", "Statistics", "StatsBase"] path = "../.." uuid = "298c2ebc-0411-48ad-af38-99e88101b606" -version = "0.1.0" +version = "0.1.1" [[ArgCheck]] git-tree-sha1 = "dedbbb2ddb876f899585c4ec4433265e3017215a" @@ -33,10 +33,10 @@ version = "2.1.0" uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" [[ArrayInterface]] -deps = ["IfElse", "LinearAlgebra", "Requires", "SparseArrays", "Static"] -git-tree-sha1 = "d84c956c4c0548b4caf0e4e96cf5b6494b5b1529" +deps = ["Compat", "IfElse", "LinearAlgebra", "Requires", "SparseArrays", "Static"] +git-tree-sha1 = "b8d49c34c3da35f220e7295659cd0bab8e739fed" uuid = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9" -version = "3.1.32" +version = "3.1.33" [[Artifacts]] uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" @@ -82,9 +82,9 @@ version = "1.11.5" [[ChainRulesCore]] deps = ["Compat", "LinearAlgebra", "SparseArrays"] -git-tree-sha1 = "4ce9393e871aca86cc457d9f66976c3da6902ea7" +git-tree-sha1 = "bd4afa1fdeec0c8b89dad3c6e92bc6e3b0fec9ce" uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" -version = "1.4.0" +version = "1.6.0" [[CloseOpenIntervals]] deps = ["ArrayInterface", "Static"] @@ -123,9 +123,9 @@ version = "0.3.0" [[Compat]] deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "SHA", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"] -git-tree-sha1 = "4866e381721b30fac8dda4c8cb1d9db45c8d2994" +git-tree-sha1 = "1a90210acd935f222ea19657f143004d2c2a1117" uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" -version = "3.37.0" +version = "3.38.0" [[CompilerSupportLibraries_jll]] deps = ["Artifacts", "Libdl"] @@ -149,9 +149,9 @@ uuid = "d38c429a-6771-53c6-b99e-75d170b6e991" version = "0.5.7" [[DataAPI]] -git-tree-sha1 = "bec2532f8adb82005476c141ec23e921fc20971b" +git-tree-sha1 = "cc70b17275652eb47bc9e5f81635981f13cea5c8" uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" -version = "1.8.0" +version = "1.9.0" [[DataStructures]] deps = ["Compat", "InteractiveUtils", "OrderedCollections"] @@ -180,9 +180,9 @@ version = "1.0.3" [[DiffRules]] deps = ["NaNMath", "Random", "SpecialFunctions"] -git-tree-sha1 = "3ed8fa7178a10d1cd0f1ca524f249ba6937490c0" +git-tree-sha1 = "7220bc21c33e990c14f4a9a319b1d242ebc5b269" uuid = "b552c78f-8df3-52c6-915a-8e097449b14b" -version = "1.3.0" +version = "1.3.1" [[Distances]] deps = ["LinearAlgebra", "Statistics", "StatsAPI"] @@ -242,9 +242,9 @@ version = "0.4.7" [[FillArrays]] deps = ["LinearAlgebra", "Random", "SparseArrays", "Statistics"] -git-tree-sha1 = "caf289224e622f518c9dbfe832cdafa17d7c80a6" +git-tree-sha1 = "7f6ad1a7f4621b4ab8e554133dade99ebc6e7221" uuid = "1a297f60-69ca-5386-bcde-b61e274b549b" -version = "0.12.4" +version = "0.12.5" [[FiniteDiff]] deps = ["ArrayInterface", "LinearAlgebra", "Requires", "SparseArrays", "StaticArrays"] @@ -311,9 +311,9 @@ version = "0.2.0" [[GR]] deps = ["Base64", "DelimitedFiles", "GR_jll", "HTTP", "JSON", "Libdl", "LinearAlgebra", "Pkg", "Printf", "Random", "Serialization", "Sockets", "Test", "UUIDs"] -git-tree-sha1 = "182da592436e287758ded5be6e32c406de3a2e47" +git-tree-sha1 = "c2178cfbc0a5a552e16d097fae508f2024de61a3" uuid = "28b8d3ca-fb5f-59d9-8090-bfdbd6d07a71" -version = "0.58.1" +version = "0.59.0" [[GR_jll]] deps = ["Artifacts", "Bzip2_jll", "Cairo_jll", "FFMPEG_jll", "Fontconfig_jll", "GLFW_jll", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Libtiff_jll", "Pixman_jll", "Pkg", "Qt5Base_jll", "Zlib_jll", "libpng_jll"] @@ -581,9 +581,9 @@ uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" [[LoopVectorization]] deps = ["ArrayInterface", "CPUSummary", "CloseOpenIntervals", "DocStringExtensions", "HostCPUFeatures", "IfElse", "LayoutPointers", "LinearAlgebra", "OffsetArrays", "PolyesterWeave", "Requires", "SLEEFPirates", "Static", "ThreadingUtilities", "UnPack", "VectorizationBase"] -git-tree-sha1 = "d469fcf148475a74c221f14d42ee75da7ccb3b4e" +git-tree-sha1 = "4804192466f4d370ca19c9957dfb3d919e6ef77e" uuid = "bdcacae8-1622-11e9-2a5c-532679323890" -version = "0.12.73" +version = "0.12.77" [[MacroTools]] deps = ["Markdown", "Random"] @@ -648,15 +648,15 @@ uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" [[NonlinearSolve]] deps = ["ArrayInterface", "FiniteDiff", "ForwardDiff", "IterativeSolvers", "LinearAlgebra", "RecursiveArrayTools", "RecursiveFactorization", "Reexport", "SciMLBase", "Setfield", "StaticArrays", "UnPack"] -git-tree-sha1 = "35585534c0c79c161241f2e65e759a11a79d25d0" +git-tree-sha1 = "e9ffc92217b8709e0cf7b8808f6223a4a0936c95" uuid = "8913a72c-1f9b-4ce2-8d82-65094dcecaec" -version = "0.3.10" +version = "0.3.11" [[OffsetArrays]] deps = ["Adapt"] -git-tree-sha1 = "c870a0d713b51e4b49be6432eff0e26a4325afee" +git-tree-sha1 = "c0e9e582987d36d5a61e650e6e543b9e44d9914b" uuid = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" -version = "1.10.6" +version = "1.10.7" [[Ogg_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] @@ -664,6 +664,10 @@ git-tree-sha1 = "7937eda4681660b4d6aeeecc2f7e1c81c8ee4e2f" uuid = "e7412a2a-1a6e-54c0-be00-318e2571c051" version = "1.3.5+0" +[[OpenLibm_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "05823500-19ac-5b8b-9628-191a04bc5112" + [[OpenSSL_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] git-tree-sha1 = "15003dcb7d8db3c6c857fda14891a539a8f2705a" @@ -713,15 +717,15 @@ version = "0.3.8" [[Parameters]] deps = ["OrderedCollections", "UnPack"] -git-tree-sha1 = "2276ac65f1e236e0a6ea70baff3f62ad4c625345" +git-tree-sha1 = "34c0e9ad262e5f7fc75b10a9952ca7692cfc5fbe" uuid = "d96e819e-fc66-5662-9728-84c9c7592b0a" -version = "0.12.2" +version = "0.12.3" [[Parsers]] deps = ["Dates"] -git-tree-sha1 = "438d35d2d95ae2c5e8780b330592b6de8494e779" +git-tree-sha1 = "9d8c00ef7a8d110787ff6f170579846f776133a9" uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" -version = "2.0.3" +version = "2.0.4" [[Pixman_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] @@ -747,15 +751,15 @@ version = "1.0.14" [[Plots]] deps = ["Base64", "Contour", "Dates", "Downloads", "FFMPEG", "FixedPointNumbers", "GR", "GeometryBasics", "JSON", "Latexify", "LinearAlgebra", "Measures", "NaNMath", "PlotThemes", "PlotUtils", "Printf", "REPL", "Random", "RecipesBase", "RecipesPipeline", "Reexport", "Requires", "Scratch", "Showoff", "SparseArrays", "Statistics", "StatsBase", "UUIDs"] -git-tree-sha1 = "2dbafeadadcf7dadff20cd60046bba416b4912be" +git-tree-sha1 = "457b13497a3ea4deb33d273a6a5ea15c25c0ebd9" uuid = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" -version = "1.21.3" +version = "1.22.2" [[Polyester]] -deps = ["ArrayInterface", "BitTwiddlingConvenienceFunctions", "CPUSummary", "IfElse", "ManualMemory", "Requires", "Static", "StrideArraysCore", "ThreadingUtilities"] -git-tree-sha1 = "21d8a7163d0f3972ade36ca2b5a0e8a27ac96842" +deps = ["ArrayInterface", "BitTwiddlingConvenienceFunctions", "CPUSummary", "IfElse", "ManualMemory", "PolyesterWeave", "Requires", "Static", "StrideArraysCore", "ThreadingUtilities"] +git-tree-sha1 = "74d358e649e0450cb5d3ff54ca7c8d806ed62765" uuid = "f517fe37-dbe3-4b94-8317-1923a5111588" -version = "0.4.4" +version = "0.5.1" [[PolyesterWeave]] deps = ["BitTwiddlingConvenienceFunctions", "CPUSummary", "IfElse", "Static", "ThreadingUtilities"] @@ -787,9 +791,9 @@ version = "5.15.3+0" [[QuadGK]] deps = ["DataStructures", "LinearAlgebra"] -git-tree-sha1 = "12fbe86da16df6679be7521dfb39fbc861e1dc7b" +git-tree-sha1 = "78aadffb3efd2155af139781b8a8df1ef279ea39" uuid = "1fd47b50-473d-5c70-9696-f719f8f3bcdc" -version = "2.4.1" +version = "2.4.2" [[REPL]] deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] @@ -806,9 +810,9 @@ version = "1.1.2" [[RecipesPipeline]] deps = ["Dates", "NaNMath", "PlotUtils", "RecipesBase"] -git-tree-sha1 = "d4491becdc53580c6dadb0f6249f90caae888554" +git-tree-sha1 = "7ad0dfa8d03b7bcf8c597f59f5292801730c55b8" uuid = "01d81517-befc-4cb6-b9ec-a95719d0359c" -version = "0.4.0" +version = "0.4.1" [[RecursiveArrayTools]] deps = ["ArrayInterface", "ChainRulesCore", "DocStringExtensions", "LinearAlgebra", "RecipesBase", "Requires", "StaticArrays", "Statistics", "ZygoteRules"] @@ -855,15 +859,15 @@ version = "0.1.0" [[SLEEFPirates]] deps = ["IfElse", "Static", "VectorizationBase"] -git-tree-sha1 = "947491c30d4293bebb00781bcaf787ba09e7c20d" +git-tree-sha1 = "2e8150c7d2a14ac68537c7aac25faa6577aff046" uuid = "476501e8-09a2-5ece-8869-fb82de89a1fa" -version = "0.6.26" +version = "0.6.27" [[SciMLBase]] deps = ["ArrayInterface", "CommonSolve", "ConstructionBase", "Distributed", "DocStringExtensions", "IteratorInterfaceExtensions", "LinearAlgebra", "Logging", "RecipesBase", "RecursiveArrayTools", "StaticArrays", "Statistics", "Tables", "TreeViews"] -git-tree-sha1 = "ff686e0c79dbe91767f4c1e44257621a5455b1c6" +git-tree-sha1 = "91e29a2bb257a4b992c48f35084064578b87d364" uuid = "0bca4576-84f4-4d90-8ffe-ffa030f20462" -version = "1.18.7" +version = "1.19.0" [[Scratch]] deps = ["Dates"] @@ -876,9 +880,9 @@ uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" [[Setfield]] deps = ["ConstructionBase", "Future", "MacroTools", "Requires"] -git-tree-sha1 = "fca29e68c5062722b5b4435594c3d1ba557072a3" +git-tree-sha1 = "def0718ddbabeb5476e51e5a43609bee889f285d" uuid = "efcf1570-3423-57d1-acb7-fd33fddbac46" -version = "0.7.1" +version = "0.8.0" [[SharedArrays]] deps = ["Distributed", "Mmap", "Random", "Serialization"] @@ -904,10 +908,10 @@ deps = ["LinearAlgebra", "Random"] uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" [[SpecialFunctions]] -deps = ["ChainRulesCore", "LogExpFunctions", "OpenSpecFun_jll"] -git-tree-sha1 = "a322a9493e49c5f3a10b50df3aedaf1cdb3244b7" +deps = ["ChainRulesCore", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"] +git-tree-sha1 = "ad42c30a6204c74d264692e633133dcea0e8b14e" uuid = "276daf66-3868-5448-9aa4-cd146d93841b" -version = "1.6.1" +version = "1.6.2" [[Static]] deps = ["IfElse"] @@ -950,9 +954,9 @@ version = "0.2.4" [[StructArrays]] deps = ["Adapt", "DataAPI", "StaticArrays", "Tables"] -git-tree-sha1 = "f41020e84127781af49fc12b7e92becd7f5dd0ba" +git-tree-sha1 = "2ce41e0d042c60ecd131e9fb7154a3bfadbf50d3" uuid = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" -version = "0.6.2" +version = "0.6.3" [[SuiteSparse]] deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"] @@ -1002,9 +1006,9 @@ version = "0.3.0" [[TriangularSolve]] deps = ["CloseOpenIntervals", "IfElse", "LayoutPointers", "LinearAlgebra", "LoopVectorization", "Polyester", "Static", "VectorizationBase"] -git-tree-sha1 = "1eed054a58d9332adc731103fe47dad2ad1a0adf" +git-tree-sha1 = "ed55426a514db35f58d36c3812aae89cfc057401" uuid = "d5829a12-d9aa-46ab-831f-fb7c9ab06edf" -version = "0.1.5" +version = "0.1.6" [[URIs]] git-tree-sha1 = "97bbe755a53fe859669cd907f2d96aee8d2c1355" @@ -1025,9 +1029,9 @@ uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" [[VectorizationBase]] deps = ["ArrayInterface", "CPUSummary", "HostCPUFeatures", "Hwloc", "IfElse", "LayoutPointers", "Libdl", "LinearAlgebra", "SIMDTypes", "Static"] -git-tree-sha1 = "43c605e008ac67adb672ef08721d4720dfe2ad41" +git-tree-sha1 = "3e2385f4ec895e694c24a1d5aba58cb6d27cf8b6" uuid = "3d5dd08c-fd9d-11e8-17fa-ed2836048c2f" -version = "0.21.7" +version = "0.21.10" [[Wayland_jll]] deps = ["Artifacts", "Expat_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Pkg", "XML2_jll"] @@ -1191,9 +1195,9 @@ version = "1.5.0+0" [[Zygote]] deps = ["AbstractFFTs", "ChainRules", "ChainRulesCore", "DiffRules", "Distributed", "FillArrays", "ForwardDiff", "IRTools", "InteractiveUtils", "LinearAlgebra", "MacroTools", "NaNMath", "Random", "Requires", "SpecialFunctions", "Statistics", "ZygoteRules"] -git-tree-sha1 = "ffbf36ba9cd8476347486a013c93590b910a4855" +git-tree-sha1 = "4b799addc63aa77ad4112cede8086564d9068511" uuid = "e88e6eb3-aa80-5325-afca-941959d7151f" -version = "0.6.21" +version = "0.6.22" [[ZygoteRules]] deps = ["MacroTools"] diff --git a/examples/c-comparisons/Manifest.toml b/examples/c-comparisons/Manifest.toml new file mode 100644 index 00000000..c596c51e --- /dev/null +++ b/examples/c-comparisons/Manifest.toml @@ -0,0 +1,1256 @@ +# This file is machine-generated - editing it directly is not advised + +[[AbstractFFTs]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "485ee0867925449198280d4af84bdb46a2a404d0" +uuid = "621f4979-c628-5d54-868e-fcf4e3e8185c" +version = "1.0.1" + +[[AbstractGPs]] +deps = ["ChainRulesCore", "Distributions", "FillArrays", "IrrationalConstants", "KernelFunctions", "LinearAlgebra", "Random", "RecipesBase", "Reexport", "Statistics", "StatsBase", "Test"] +git-tree-sha1 = "80b6c9734d00ae7518e65a0e1063772a050d04a0" +uuid = "99985d1d-32ba-4be9-9821-2ec096f28918" +version = "0.5.1" + +[[Adapt]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "84918055d15b3114ede17ac6a7182f68870c16f7" +uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" +version = "3.3.1" + +[[ApproximateGPs]] +deps = ["AbstractGPs", "ChainRulesCore", "Distributions", "FastGaussQuadrature", "FillArrays", "ForwardDiff", "GPLikelihoods", "KLDivergences", "LinearAlgebra", "QuadGK", "Reexport", "SpecialFunctions", "Statistics", "StatsBase"] +path = "../.." +uuid = "298c2ebc-0411-48ad-af38-99e88101b606" +version = "0.1.1" + +[[ArgCheck]] +git-tree-sha1 = "dedbbb2ddb876f899585c4ec4433265e3017215a" +uuid = "dce04be8-c92d-5529-be00-80e4d2c0e197" +version = "2.1.0" + +[[ArgTools]] +uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" + +[[ArrayInterface]] +deps = ["Compat", "IfElse", "LinearAlgebra", "Requires", "SparseArrays", "Static"] +git-tree-sha1 = "b8d49c34c3da35f220e7295659cd0bab8e739fed" +uuid = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9" +version = "3.1.33" + +[[Artifacts]] +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" + +[[Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" + +[[Bijectors]] +deps = ["ArgCheck", "ChainRulesCore", "Compat", "Distributions", "Functors", "IrrationalConstants", "LinearAlgebra", "LogExpFunctions", "MappedArrays", "NonlinearSolve", "Random", "Reexport", "Requires", "SparseArrays", "Statistics"] +git-tree-sha1 = "dca5e02c9426b2f8ce86d8e723d0702ff33df234" +uuid = "76274a88-744f-5084-9051-94815aaf08c4" +version = "0.9.8" + +[[BitTwiddlingConvenienceFunctions]] +deps = ["Static"] +git-tree-sha1 = "652aab0fc0d6d4db4cc726425cadf700e9f473f1" +uuid = "62783981-4cbd-42fc-bca8-16325de8dc4b" +version = "0.1.0" + +[[Bzip2_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "19a35467a82e236ff51bc17a3a44b69ef35185a2" +uuid = "6e34b625-4abd-537c-b88f-471c36dfa7a0" +version = "1.0.8+0" + +[[CPUSummary]] +deps = ["Hwloc", "IfElse", "Static"] +git-tree-sha1 = "ed720e2622820bf584d4ad90e6fcb93d95170b44" +uuid = "2a0fbf3d-bb9c-48f3-b0a9-814d99fd7ab9" +version = "0.1.3" + +[[Cairo_jll]] +deps = ["Artifacts", "Bzip2_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "JLLWrappers", "LZO_jll", "Libdl", "Pixman_jll", "Pkg", "Xorg_libXext_jll", "Xorg_libXrender_jll", "Zlib_jll", "libpng_jll"] +git-tree-sha1 = "f2202b55d816427cd385a9a4f3ffb226bee80f99" +uuid = "83423d85-b0ee-5818-9007-b63ccbeb887a" +version = "1.16.1+0" + +[[ChainRules]] +deps = ["ChainRulesCore", "Compat", "LinearAlgebra", "Random", "Statistics"] +git-tree-sha1 = "d88340ab502af66cfffc821e70ae72f7dbdce645" +uuid = "082447d4-558c-5d27-93f4-14fc19e9eca2" +version = "1.11.5" + +[[ChainRulesCore]] +deps = ["Compat", "LinearAlgebra", "SparseArrays"] +git-tree-sha1 = "bd4afa1fdeec0c8b89dad3c6e92bc6e3b0fec9ce" +uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" +version = "1.6.0" + +[[CloseOpenIntervals]] +deps = ["ArrayInterface", "Static"] +git-tree-sha1 = "ce9c0d07ed6e1a4fecd2df6ace144cbd29ba6f37" +uuid = "fb6a15b2-703c-40df-9091-08a04967cfa9" +version = "0.1.2" + +[[ColorSchemes]] +deps = ["ColorTypes", "Colors", "FixedPointNumbers", "Random"] +git-tree-sha1 = "9995eb3977fbf67b86d0a0a0508e83017ded03f2" +uuid = "35d6a980-a343-548e-a6ea-1d62b119f2f4" +version = "3.14.0" + +[[ColorTypes]] +deps = ["FixedPointNumbers", "Random"] +git-tree-sha1 = "024fe24d83e4a5bf5fc80501a314ce0d1aa35597" +uuid = "3da002f7-5984-5a60-b8a6-cbb66c0b333f" +version = "0.11.0" + +[[Colors]] +deps = ["ColorTypes", "FixedPointNumbers", "Reexport"] +git-tree-sha1 = "417b0ed7b8b838aa6ca0a87aadf1bb9eb111ce40" +uuid = "5ae59095-9a9b-59fe-a467-6f913c188581" +version = "0.12.8" + +[[CommonSolve]] +git-tree-sha1 = "68a0743f578349ada8bc911a5cbd5a2ef6ed6d1f" +uuid = "38540f10-b2f7-11e9-35d8-d573e4eb0ff2" +version = "0.2.0" + +[[CommonSubexpressions]] +deps = ["MacroTools", "Test"] +git-tree-sha1 = "7b8a93dba8af7e3b42fecabf646260105ac373f7" +uuid = "bbf7d656-a473-5ed7-a52c-81e309532950" +version = "0.3.0" + +[[Compat]] +deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "SHA", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"] +git-tree-sha1 = "1a90210acd935f222ea19657f143004d2c2a1117" +uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" +version = "3.38.0" + +[[CompilerSupportLibraries_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" + +[[CompositionsBase]] +git-tree-sha1 = "455419f7e328a1a2493cabc6428d79e951349769" +uuid = "a33af91c-f02d-484b-be07-31d278c5ca2b" +version = "0.1.1" + +[[ConstructionBase]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "f74e9d5388b8620b4cee35d4c5a618dd4dc547f4" +uuid = "187b0558-2788-49d3-abe0-74a17ed4e7c9" +version = "1.3.0" + +[[Contour]] +deps = ["StaticArrays"] +git-tree-sha1 = "9f02045d934dc030edad45944ea80dbd1f0ebea7" +uuid = "d38c429a-6771-53c6-b99e-75d170b6e991" +version = "0.5.7" + +[[DataAPI]] +git-tree-sha1 = "cc70b17275652eb47bc9e5f81635981f13cea5c8" +uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" +version = "1.9.0" + +[[DataStructures]] +deps = ["Compat", "InteractiveUtils", "OrderedCollections"] +git-tree-sha1 = "7d9d316f04214f7efdbb6398d545446e246eff02" +uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" +version = "0.18.10" + +[[DataValueInterfaces]] +git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6" +uuid = "e2d170a0-9d28-54be-80f0-106bbe20a464" +version = "1.0.0" + +[[Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" + +[[DelimitedFiles]] +deps = ["Mmap"] +uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab" + +[[DiffResults]] +deps = ["StaticArrays"] +git-tree-sha1 = "c18e98cba888c6c25d1c3b048e4b3380ca956805" +uuid = "163ba53b-c6d8-5494-b064-1a9d43ac40c5" +version = "1.0.3" + +[[DiffRules]] +deps = ["NaNMath", "Random", "SpecialFunctions"] +git-tree-sha1 = "7220bc21c33e990c14f4a9a319b1d242ebc5b269" +uuid = "b552c78f-8df3-52c6-915a-8e097449b14b" +version = "1.3.1" + +[[Distances]] +deps = ["LinearAlgebra", "Statistics", "StatsAPI"] +git-tree-sha1 = "9f46deb4d4ee4494ffb5a40a27a2aced67bdd838" +uuid = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7" +version = "0.10.4" + +[[Distributed]] +deps = ["Random", "Serialization", "Sockets"] +uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" + +[[Distributions]] +deps = ["ChainRulesCore", "FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "SparseArrays", "SpecialFunctions", "Statistics", "StatsBase", "StatsFuns"] +git-tree-sha1 = "f4efaa4b5157e0cdb8283ae0b5428bc9208436ed" +uuid = "31c24e10-a181-5473-b8eb-7969acd0382f" +version = "0.25.16" + +[[DocStringExtensions]] +deps = ["LibGit2"] +git-tree-sha1 = "a32185f5428d3986f47c2ab78b1f216d5e6cc96f" +uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" +version = "0.8.5" + +[[Downloads]] +deps = ["ArgTools", "LibCURL", "NetworkOptions"] +uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" + +[[EarCut_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "3f3a2501fa7236e9b911e0f7a588c657e822bb6d" +uuid = "5ae413db-bbd1-5e63-b57d-d24a61df00f5" +version = "2.2.3+0" + +[[Expat_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "b3bfd02e98aedfa5cf885665493c5598c350cd2f" +uuid = "2e619515-83b5-522b-bb60-26c02a35a201" +version = "2.2.10+0" + +[[FFMPEG]] +deps = ["FFMPEG_jll"] +git-tree-sha1 = "b57e3acbe22f8484b4b5ff66a7499717fe1a9cc8" +uuid = "c87230d0-a227-11e9-1b43-d7ebe4e7570a" +version = "0.4.1" + +[[FFMPEG_jll]] +deps = ["Artifacts", "Bzip2_jll", "FreeType2_jll", "FriBidi_jll", "JLLWrappers", "LAME_jll", "Libdl", "Ogg_jll", "OpenSSL_jll", "Opus_jll", "Pkg", "Zlib_jll", "libass_jll", "libfdk_aac_jll", "libvorbis_jll", "x264_jll", "x265_jll"] +git-tree-sha1 = "d8a578692e3077ac998b50c0217dfd67f21d1e5f" +uuid = "b22a6f82-2f65-5046-a5b2-351ab43fb4e5" +version = "4.4.0+0" + +[[FastGaussQuadrature]] +deps = ["LinearAlgebra", "SpecialFunctions", "StaticArrays"] +git-tree-sha1 = "5829b25887e53fb6730a9df2ff89ed24baa6abf6" +uuid = "442a2c76-b920-505d-bb47-c5924d526838" +version = "0.4.7" + +[[FillArrays]] +deps = ["LinearAlgebra", "Random", "SparseArrays", "Statistics"] +git-tree-sha1 = "7f6ad1a7f4621b4ab8e554133dade99ebc6e7221" +uuid = "1a297f60-69ca-5386-bcde-b61e274b549b" +version = "0.12.5" + +[[FiniteDiff]] +deps = ["ArrayInterface", "LinearAlgebra", "Requires", "SparseArrays", "StaticArrays"] +git-tree-sha1 = "8b3c09b56acaf3c0e581c66638b85c8650ee9dca" +uuid = "6a86dc24-6348-571c-b903-95158fe2bd41" +version = "2.8.1" + +[[FixedPointNumbers]] +deps = ["Statistics"] +git-tree-sha1 = "335bfdceacc84c5cdf16aadc768aa5ddfc5383cc" +uuid = "53c48c17-4a7d-5ca2-90c5-79b7896eea93" +version = "0.8.4" + +[[Fontconfig_jll]] +deps = ["Artifacts", "Bzip2_jll", "Expat_jll", "FreeType2_jll", "JLLWrappers", "Libdl", "Libuuid_jll", "Pkg", "Zlib_jll"] +git-tree-sha1 = "21efd19106a55620a188615da6d3d06cd7f6ee03" +uuid = "a3f928ae-7b40-5064-980b-68af3947d34b" +version = "2.13.93+0" + +[[Formatting]] +deps = ["Printf"] +git-tree-sha1 = "8339d61043228fdd3eb658d86c926cb282ae72a8" +uuid = "59287772-0a20-5a39-b81b-1366585eb4c0" +version = "0.4.2" + +[[ForwardDiff]] +deps = ["CommonSubexpressions", "DiffResults", "DiffRules", "LinearAlgebra", "NaNMath", "Printf", "Random", "SpecialFunctions", "StaticArrays"] +git-tree-sha1 = "b5e930ac60b613ef3406da6d4f42c35d8dc51419" +uuid = "f6369f11-7733-5829-9624-2563aa707210" +version = "0.10.19" + +[[FreeType2_jll]] +deps = ["Artifacts", "Bzip2_jll", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"] +git-tree-sha1 = "87eb71354d8ec1a96d4a7636bd57a7347dde3ef9" +uuid = "d7e528f0-a631-5988-bf34-fe36492bcfd7" +version = "2.10.4+0" + +[[FriBidi_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "aa31987c2ba8704e23c6c8ba8a4f769d5d7e4f91" +uuid = "559328eb-81f9-559d-9380-de523a88c83c" +version = "1.0.10+0" + +[[Functors]] +git-tree-sha1 = "e2727f02325451f6b24445cd83bfa9aaac19cbe7" +uuid = "d9f16b24-f501-4c13-a1f2-28368ffc5196" +version = "0.2.5" + +[[Future]] +deps = ["Random"] +uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" + +[[GLFW_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Libglvnd_jll", "Pkg", "Xorg_libXcursor_jll", "Xorg_libXi_jll", "Xorg_libXinerama_jll", "Xorg_libXrandr_jll"] +git-tree-sha1 = "dba1e8614e98949abfa60480b13653813d8f0157" +uuid = "0656b61e-2033-5cc2-a64a-77c0f6c09b89" +version = "3.3.5+0" + +[[GPLikelihoods]] +deps = ["Distributions", "Functors", "LinearAlgebra", "Random", "StatsFuns"] +git-tree-sha1 = "e07acc5ca79ead40c8cf0338c9357fc7fb90eea1" +uuid = "6031954c-0455-49d7-b3b9-3e1c99afaf40" +version = "0.2.0" + +[[GR]] +deps = ["Base64", "DelimitedFiles", "GR_jll", "HTTP", "JSON", "Libdl", "LinearAlgebra", "Pkg", "Printf", "Random", "Serialization", "Sockets", "Test", "UUIDs"] +git-tree-sha1 = "c2178cfbc0a5a552e16d097fae508f2024de61a3" +uuid = "28b8d3ca-fb5f-59d9-8090-bfdbd6d07a71" +version = "0.59.0" + +[[GR_jll]] +deps = ["Artifacts", "Bzip2_jll", "Cairo_jll", "FFMPEG_jll", "Fontconfig_jll", "GLFW_jll", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Libtiff_jll", "Pixman_jll", "Pkg", "Qt5Base_jll", "Zlib_jll", "libpng_jll"] +git-tree-sha1 = "ef49a187604f865f4708c90e3f431890724e9012" +uuid = "d2c73de3-f751-5644-a686-071e5b155ba9" +version = "0.59.0+0" + +[[GeometryBasics]] +deps = ["EarCut_jll", "IterTools", "LinearAlgebra", "StaticArrays", "StructArrays", "Tables"] +git-tree-sha1 = "58bcdf5ebc057b085e58d95c138725628dd7453c" +uuid = "5c1252a2-5f33-56bf-86c9-59e7332b4326" +version = "0.4.1" + +[[Gettext_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Libiconv_jll", "Pkg", "XML2_jll"] +git-tree-sha1 = "9b02998aba7bf074d14de89f9d37ca24a1a0b046" +uuid = "78b55507-aeef-58d4-861c-77aaff3498b1" +version = "0.21.0+0" + +[[Glib_jll]] +deps = ["Artifacts", "Gettext_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Libiconv_jll", "Libmount_jll", "PCRE_jll", "Pkg", "Zlib_jll"] +git-tree-sha1 = "7bf67e9a481712b3dbe9cb3dac852dc4b1162e02" +uuid = "7746bdde-850d-59dc-9ae8-88ece973131d" +version = "2.68.3+0" + +[[Graphite2_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "344bf40dcab1073aca04aa0df4fb092f920e4011" +uuid = "3b182d85-2403-5c21-9c21-1e1f0cc25472" +version = "1.3.14+0" + +[[Grisu]] +git-tree-sha1 = "53bb909d1151e57e2484c3d1b53e19552b887fb2" +uuid = "42e2da0e-8278-4e71-bc24-59509adca0fe" +version = "1.0.2" + +[[HTTP]] +deps = ["Base64", "Dates", "IniFile", "Logging", "MbedTLS", "NetworkOptions", "Sockets", "URIs"] +git-tree-sha1 = "60ed5f1643927479f845b0135bb369b031b541fa" +uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3" +version = "0.9.14" + +[[HarfBuzz_jll]] +deps = ["Artifacts", "Cairo_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "Graphite2_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Pkg"] +git-tree-sha1 = "8a954fed8ac097d5be04921d595f741115c1b2ad" +uuid = "2e76f6c2-a576-52d4-95c1-20adfe4de566" +version = "2.8.1+0" + +[[HostCPUFeatures]] +deps = ["BitTwiddlingConvenienceFunctions", "IfElse", "Libdl", "Static"] +git-tree-sha1 = "3169c8b31863f9a409be1d17693751314241e3eb" +uuid = "3e5b6fbb-0976-4d2c-9146-d79de83f2fb0" +version = "0.1.4" + +[[Hwloc]] +deps = ["Hwloc_jll"] +git-tree-sha1 = "92d99146066c5c6888d5a3abc871e6a214388b91" +uuid = "0e44f5e4-bd66-52a0-8798-143a42290a1d" +version = "2.0.0" + +[[Hwloc_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "3395d4d4aeb3c9d31f5929d32760d8baeee88aaf" +uuid = "e33a78d0-f292-5ffc-b300-72abe9b543c8" +version = "2.5.0+0" + +[[IOCapture]] +deps = ["Logging", "Random"] +git-tree-sha1 = "f7be53659ab06ddc986428d3a9dcc95f6fa6705a" +uuid = "b5f81e59-6552-4d32-b1f0-c071b021bf89" +version = "0.2.2" + +[[IRTools]] +deps = ["InteractiveUtils", "MacroTools", "Test"] +git-tree-sha1 = "95215cd0076a150ef46ff7928892bc341864c73c" +uuid = "7869d1d1-7146-5819-86e3-90919afe41df" +version = "0.4.3" + +[[IfElse]] +git-tree-sha1 = "28e837ff3e7a6c3cdb252ce49fb412c8eb3caeef" +uuid = "615f187c-cbe4-4ef1-ba3b-2fcf58d6d173" +version = "0.1.0" + +[[IniFile]] +deps = ["Test"] +git-tree-sha1 = "098e4d2c533924c921f9f9847274f2ad89e018b8" +uuid = "83e8ac13-25f8-5344-8a64-a9f2b223428f" +version = "0.5.0" + +[[InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" + +[[IrrationalConstants]] +git-tree-sha1 = "f76424439413893a832026ca355fe273e93bce94" +uuid = "92d709cd-6900-40b7-9082-c6be49f344b6" +version = "0.1.0" + +[[IterTools]] +git-tree-sha1 = "05110a2ab1fc5f932622ffea2a003221f4782c18" +uuid = "c8e1da08-722c-5040-9ed9-7db0dc04731e" +version = "1.3.0" + +[[IterativeSolvers]] +deps = ["LinearAlgebra", "Printf", "Random", "RecipesBase", "SparseArrays"] +git-tree-sha1 = "1a8c6237e78b714e901e406c096fc8a65528af7d" +uuid = "42fd0dbc-a981-5370-80f2-aaf504508153" +version = "0.9.1" + +[[IteratorInterfaceExtensions]] +git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856" +uuid = "82899510-4779-5014-852e-03e436cf321d" +version = "1.0.0" + +[[JLLWrappers]] +deps = ["Preferences"] +git-tree-sha1 = "642a199af8b68253517b80bd3bfd17eb4e84df6e" +uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" +version = "1.3.0" + +[[JSON]] +deps = ["Dates", "Mmap", "Parsers", "Unicode"] +git-tree-sha1 = "8076680b162ada2a031f707ac7b4953e30667a37" +uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +version = "0.21.2" + +[[JpegTurbo_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "d735490ac75c5cb9f1b00d8b5509c11984dc6943" +uuid = "aacddb02-875f-59d6-b918-886e6ef4fbf8" +version = "2.1.0+0" + +[[KLDivergences]] +deps = ["ChainRulesCore", "Distances", "Distributions", "LinearAlgebra", "PDMats", "SpecialFunctions", "StatsBase"] +git-tree-sha1 = "b4663db8fb56053b1d1a2af80533b645eba9583a" +uuid = "3c9cd921-3d3f-41e2-830c-e020174918cc" +version = "0.2.1" + +[[KernelFunctions]] +deps = ["ChainRulesCore", "Compat", "CompositionsBase", "Distances", "FillArrays", "Functors", "IrrationalConstants", "LinearAlgebra", "LogExpFunctions", "Random", "Requires", "SpecialFunctions", "StatsBase", "TensorCore", "Test", "ZygoteRules"] +git-tree-sha1 = "3b7fceeab37b650c280eb072ffe2b868b03a5423" +uuid = "ec8451be-7e33-11e9-00cf-bbf324bd1392" +version = "0.10.17" + +[[LAME_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "f6250b16881adf048549549fba48b1161acdac8c" +uuid = "c1c5ebd0-6772-5130-a774-d5fcae4a789d" +version = "3.100.1+0" + +[[LZO_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "e5b909bcf985c5e2605737d2ce278ed791b89be6" +uuid = "dd4b983a-f0e5-5f8d-a1b7-129d4a5fb1ac" +version = "2.10.1+0" + +[[LaTeXStrings]] +git-tree-sha1 = "c7f1c695e06c01b95a67f0cd1d34994f3e7db104" +uuid = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" +version = "1.2.1" + +[[Latexify]] +deps = ["Formatting", "InteractiveUtils", "LaTeXStrings", "MacroTools", "Markdown", "Printf", "Requires"] +git-tree-sha1 = "a4b12a1bd2ebade87891ab7e36fdbce582301a92" +uuid = "23fbe1c1-3f47-55db-b15f-69d7ec21a316" +version = "0.15.6" + +[[LayoutPointers]] +deps = ["ArrayInterface", "LinearAlgebra", "ManualMemory", "SIMDTypes", "Static"] +git-tree-sha1 = "d2bda6aa0b03ce6f141a2dc73d0bcb7070131adc" +uuid = "10f19ff3-798f-405d-979b-55457f8fc047" +version = "0.1.3" + +[[LibCURL]] +deps = ["LibCURL_jll", "MozillaCACerts_jll"] +uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" + +[[LibCURL_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] +uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" + +[[LibGit2]] +deps = ["Base64", "NetworkOptions", "Printf", "SHA"] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" + +[[LibSSH2_jll]] +deps = ["Artifacts", "Libdl", "MbedTLS_jll"] +uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" + +[[Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" + +[[Libffi_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "761a393aeccd6aa92ec3515e428c26bf99575b3b" +uuid = "e9f186c6-92d2-5b65-8a66-fee21dc1b490" +version = "3.2.2+0" + +[[Libgcrypt_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Libgpg_error_jll", "Pkg"] +git-tree-sha1 = "64613c82a59c120435c067c2b809fc61cf5166ae" +uuid = "d4300ac3-e22c-5743-9152-c294e39db1e4" +version = "1.8.7+0" + +[[Libglvnd_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll", "Xorg_libXext_jll"] +git-tree-sha1 = "7739f837d6447403596a75d19ed01fd08d6f56bf" +uuid = "7e76a0d4-f3c7-5321-8279-8d96eeed0f29" +version = "1.3.0+3" + +[[Libgpg_error_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "c333716e46366857753e273ce6a69ee0945a6db9" +uuid = "7add5ba3-2f88-524e-9cd5-f83b8a55f7b8" +version = "1.42.0+0" + +[[Libiconv_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "42b62845d70a619f063a7da093d995ec8e15e778" +uuid = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531" +version = "1.16.1+1" + +[[Libmount_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "9c30530bf0effd46e15e0fdcf2b8636e78cbbd73" +uuid = "4b2f31a3-9ecc-558c-b454-b3730dcb73e9" +version = "2.35.0+0" + +[[Libtiff_jll]] +deps = ["Artifacts", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Pkg", "Zlib_jll", "Zstd_jll"] +git-tree-sha1 = "340e257aada13f95f98ee352d316c3bed37c8ab9" +uuid = "89763e89-9b03-5906-acba-b20f662cd828" +version = "4.3.0+0" + +[[Libuuid_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "7f3efec06033682db852f8b3bc3c1d2b0a0ab066" +uuid = "38a345b3-de98-5d2b-a5d3-14cd9215e700" +version = "2.36.0+0" + +[[LineSearches]] +deps = ["LinearAlgebra", "NLSolversBase", "NaNMath", "Parameters", "Printf"] +git-tree-sha1 = "f27132e551e959b3667d8c93eae90973225032dd" +uuid = "d3d80556-e9d4-5f37-9878-2ab0fcc64255" +version = "7.1.1" + +[[LinearAlgebra]] +deps = ["Libdl"] +uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + +[[Literate]] +deps = ["Base64", "IOCapture", "JSON", "REPL"] +git-tree-sha1 = "bbebc3c14dbfbe76bfcbabf0937481ac84dc86ef" +uuid = "98b081ad-f1c9-55d3-8b20-4c87d4299306" +version = "2.9.3" + +[[LogExpFunctions]] +deps = ["ChainRulesCore", "DocStringExtensions", "IrrationalConstants", "LinearAlgebra"] +git-tree-sha1 = "34dc30f868e368f8a17b728a1238f3fcda43931a" +uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688" +version = "0.3.3" + +[[Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" + +[[LoopVectorization]] +deps = ["ArrayInterface", "CPUSummary", "CloseOpenIntervals", "DocStringExtensions", "HostCPUFeatures", "IfElse", "LayoutPointers", "LinearAlgebra", "OffsetArrays", "PolyesterWeave", "Requires", "SLEEFPirates", "Static", "ThreadingUtilities", "UnPack", "VectorizationBase"] +git-tree-sha1 = "4804192466f4d370ca19c9957dfb3d919e6ef77e" +uuid = "bdcacae8-1622-11e9-2a5c-532679323890" +version = "0.12.77" + +[[MacroTools]] +deps = ["Markdown", "Random"] +git-tree-sha1 = "5a5bc6bf062f0f95e62d0fe0a2d99699fed82dd9" +uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" +version = "0.5.8" + +[[ManualMemory]] +git-tree-sha1 = "9cb207b18148b2199db259adfa923b45593fe08e" +uuid = "d125e4d3-2237-4719-b19c-fa641b8a4667" +version = "0.1.6" + +[[MappedArrays]] +git-tree-sha1 = "e8b359ef06ec72e8c030463fe02efe5527ee5142" +uuid = "dbb5928d-eab1-5f90-85c2-b9b0edb7c900" +version = "0.4.1" + +[[Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" + +[[MbedTLS]] +deps = ["Dates", "MbedTLS_jll", "Random", "Sockets"] +git-tree-sha1 = "1c38e51c3d08ef2278062ebceade0e46cefc96fe" +uuid = "739be429-bea8-5141-9913-cc70e7f3736d" +version = "1.0.3" + +[[MbedTLS_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" + +[[Measures]] +git-tree-sha1 = "e498ddeee6f9fdb4551ce855a46f54dbd900245f" +uuid = "442fdcdd-2543-5da2-b0f3-8c86c306513e" +version = "0.3.1" + +[[Missings]] +deps = ["DataAPI"] +git-tree-sha1 = "bf210ce90b6c9eed32d25dbcae1ebc565df2687f" +uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28" +version = "1.0.2" + +[[Mmap]] +uuid = "a63ad114-7e13-5084-954f-fe012c677804" + +[[MozillaCACerts_jll]] +uuid = "14a3606d-f60d-562e-9121-12d972cd8159" + +[[NLSolversBase]] +deps = ["DiffResults", "Distributed", "FiniteDiff", "ForwardDiff"] +git-tree-sha1 = "144bab5b1443545bc4e791536c9f1eacb4eed06a" +uuid = "d41bc354-129a-5804-8e4c-c37616107c6c" +version = "7.8.1" + +[[NaNMath]] +git-tree-sha1 = "bfe47e760d60b82b66b61d2d44128b62e3a369fb" +uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3" +version = "0.3.5" + +[[NetworkOptions]] +uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" + +[[NonlinearSolve]] +deps = ["ArrayInterface", "FiniteDiff", "ForwardDiff", "IterativeSolvers", "LinearAlgebra", "RecursiveArrayTools", "RecursiveFactorization", "Reexport", "SciMLBase", "Setfield", "StaticArrays", "UnPack"] +git-tree-sha1 = "e9ffc92217b8709e0cf7b8808f6223a4a0936c95" +uuid = "8913a72c-1f9b-4ce2-8d82-65094dcecaec" +version = "0.3.11" + +[[OffsetArrays]] +deps = ["Adapt"] +git-tree-sha1 = "c0e9e582987d36d5a61e650e6e543b9e44d9914b" +uuid = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" +version = "1.10.7" + +[[Ogg_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "7937eda4681660b4d6aeeecc2f7e1c81c8ee4e2f" +uuid = "e7412a2a-1a6e-54c0-be00-318e2571c051" +version = "1.3.5+0" + +[[OpenLibm_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "05823500-19ac-5b8b-9628-191a04bc5112" + +[[OpenSSL_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "15003dcb7d8db3c6c857fda14891a539a8f2705a" +uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" +version = "1.1.10+0" + +[[OpenSpecFun_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "13652491f6856acfd2db29360e1bbcd4565d04f1" +uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e" +version = "0.5.5+0" + +[[Optim]] +deps = ["Compat", "FillArrays", "LineSearches", "LinearAlgebra", "NLSolversBase", "NaNMath", "Parameters", "PositiveFactorizations", "Printf", "SparseArrays", "StatsBase"] +git-tree-sha1 = "7863df65dbb2a0fa8f85fcaf0a41167640d2ebed" +uuid = "429524aa-4258-5aef-a3af-852621145aeb" +version = "1.4.1" + +[[Opus_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "51a08fb14ec28da2ec7a927c4337e4332c2a4720" +uuid = "91d4177d-7536-5919-b921-800302f37372" +version = "1.3.2+0" + +[[OrderedCollections]] +git-tree-sha1 = "85f8e6578bf1f9ee0d11e7bb1b1456435479d47c" +uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +version = "1.4.1" + +[[PCRE_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "b2a7af664e098055a7529ad1a900ded962bca488" +uuid = "2f80f16e-611a-54ab-bc61-aa92de5b98fc" +version = "8.44.0+0" + +[[PDMats]] +deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"] +git-tree-sha1 = "4dd403333bcf0909341cfe57ec115152f937d7d8" +uuid = "90014a1f-27ba-587c-ab20-58faa44d9150" +version = "0.11.1" + +[[ParameterHandling]] +deps = ["Bijectors", "ChainRulesCore", "Compat", "IterTools", "LinearAlgebra", "SparseArrays", "Test"] +git-tree-sha1 = "b454231b4559c118fe3522734c71555592445843" +uuid = "2412ca09-6db7-441c-8e3a-88d5709968c5" +version = "0.3.8" + +[[Parameters]] +deps = ["OrderedCollections", "UnPack"] +git-tree-sha1 = "34c0e9ad262e5f7fc75b10a9952ca7692cfc5fbe" +uuid = "d96e819e-fc66-5662-9728-84c9c7592b0a" +version = "0.12.3" + +[[Parsers]] +deps = ["Dates"] +git-tree-sha1 = "9d8c00ef7a8d110787ff6f170579846f776133a9" +uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" +version = "2.0.4" + +[[Pixman_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "b4f5d02549a10e20780a24fce72bea96b6329e29" +uuid = "30392449-352a-5448-841d-b1acce4e97dc" +version = "0.40.1+0" + +[[Pkg]] +deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" + +[[PlotThemes]] +deps = ["PlotUtils", "Requires", "Statistics"] +git-tree-sha1 = "a3a964ce9dc7898193536002a6dd892b1b5a6f1d" +uuid = "ccf2f8ad-2431-5c83-bf29-c5338b663b6a" +version = "2.0.1" + +[[PlotUtils]] +deps = ["ColorSchemes", "Colors", "Dates", "Printf", "Random", "Reexport", "Statistics"] +git-tree-sha1 = "2537ed3c0ed5e03896927187f5f2ee6a4ab342db" +uuid = "995b91a9-d308-5afd-9ec6-746e21dbc043" +version = "1.0.14" + +[[Plots]] +deps = ["Base64", "Contour", "Dates", "Downloads", "FFMPEG", "FixedPointNumbers", "GR", "GeometryBasics", "JSON", "Latexify", "LinearAlgebra", "Measures", "NaNMath", "PlotThemes", "PlotUtils", "Printf", "REPL", "Random", "RecipesBase", "RecipesPipeline", "Reexport", "Requires", "Scratch", "Showoff", "SparseArrays", "Statistics", "StatsBase", "UUIDs"] +git-tree-sha1 = "457b13497a3ea4deb33d273a6a5ea15c25c0ebd9" +uuid = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" +version = "1.22.2" + +[[Polyester]] +deps = ["ArrayInterface", "BitTwiddlingConvenienceFunctions", "CPUSummary", "IfElse", "ManualMemory", "PolyesterWeave", "Requires", "Static", "StrideArraysCore", "ThreadingUtilities"] +git-tree-sha1 = "74d358e649e0450cb5d3ff54ca7c8d806ed62765" +uuid = "f517fe37-dbe3-4b94-8317-1923a5111588" +version = "0.5.1" + +[[PolyesterWeave]] +deps = ["BitTwiddlingConvenienceFunctions", "CPUSummary", "IfElse", "Static", "ThreadingUtilities"] +git-tree-sha1 = "371a19bb801c1b420b29141750f3a34d6c6634b9" +uuid = "1d0040c9-8b98-4ee7-8388-3f51789ca0ad" +version = "0.1.0" + +[[PositiveFactorizations]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "17275485f373e6673f7e7f97051f703ed5b15b20" +uuid = "85a6dd25-e78a-55b7-8502-1745935b8125" +version = "0.2.4" + +[[Preferences]] +deps = ["TOML"] +git-tree-sha1 = "00cfd92944ca9c760982747e9a1d0d5d86ab1e5a" +uuid = "21216c6a-2e73-6563-6e65-726566657250" +version = "1.2.2" + +[[Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" + +[[Qt5Base_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Fontconfig_jll", "Glib_jll", "JLLWrappers", "Libdl", "Libglvnd_jll", "OpenSSL_jll", "Pkg", "Xorg_libXext_jll", "Xorg_libxcb_jll", "Xorg_xcb_util_image_jll", "Xorg_xcb_util_keysyms_jll", "Xorg_xcb_util_renderutil_jll", "Xorg_xcb_util_wm_jll", "Zlib_jll", "xkbcommon_jll"] +git-tree-sha1 = "ad368663a5e20dbb8d6dc2fddeefe4dae0781ae8" +uuid = "ea2cea3b-5b76-57ae-a6ef-0a8af62496e1" +version = "5.15.3+0" + +[[QuadGK]] +deps = ["DataStructures", "LinearAlgebra"] +git-tree-sha1 = "78aadffb3efd2155af139781b8a8df1ef279ea39" +uuid = "1fd47b50-473d-5c70-9696-f719f8f3bcdc" +version = "2.4.2" + +[[REPL]] +deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" + +[[Random]] +deps = ["Serialization"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" + +[[RecipesBase]] +git-tree-sha1 = "44a75aa7a527910ee3d1751d1f0e4148698add9e" +uuid = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" +version = "1.1.2" + +[[RecipesPipeline]] +deps = ["Dates", "NaNMath", "PlotUtils", "RecipesBase"] +git-tree-sha1 = "7ad0dfa8d03b7bcf8c597f59f5292801730c55b8" +uuid = "01d81517-befc-4cb6-b9ec-a95719d0359c" +version = "0.4.1" + +[[RecursiveArrayTools]] +deps = ["ArrayInterface", "ChainRulesCore", "DocStringExtensions", "LinearAlgebra", "RecipesBase", "Requires", "StaticArrays", "Statistics", "ZygoteRules"] +git-tree-sha1 = "00bede2eb099dcc1ddc3f9ec02180c326b420ee2" +uuid = "731186ca-8d62-57ce-b412-fbd966d074cd" +version = "2.17.2" + +[[RecursiveFactorization]] +deps = ["LinearAlgebra", "LoopVectorization", "Polyester", "StrideArraysCore", "TriangularSolve"] +git-tree-sha1 = "575c18c6b00ce409f75d96fefe33ebe01575457a" +uuid = "f2c3362d-daeb-58d1-803e-2bc74f2840b4" +version = "0.2.4" + +[[Reexport]] +git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" +uuid = "189a3867-3050-52da-a836-e630ba90ab69" +version = "1.2.2" + +[[Requires]] +deps = ["UUIDs"] +git-tree-sha1 = "4036a3bd08ac7e968e27c203d45f5fff15020621" +uuid = "ae029012-a4dd-5104-9daa-d747884805df" +version = "1.1.3" + +[[Rmath]] +deps = ["Random", "Rmath_jll"] +git-tree-sha1 = "bf3188feca147ce108c76ad82c2792c57abe7b1f" +uuid = "79098fc4-a85e-5d69-aa6a-4863f24498fa" +version = "0.7.0" + +[[Rmath_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "68db32dff12bb6127bac73c209881191bf0efbb7" +uuid = "f50d1b31-88e8-58de-be2c-1cc44531875f" +version = "0.3.0+0" + +[[SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" + +[[SIMDTypes]] +git-tree-sha1 = "330289636fb8107c5f32088d2741e9fd7a061a5c" +uuid = "94e857df-77ce-4151-89e5-788b33177be4" +version = "0.1.0" + +[[SLEEFPirates]] +deps = ["IfElse", "Static", "VectorizationBase"] +git-tree-sha1 = "2e8150c7d2a14ac68537c7aac25faa6577aff046" +uuid = "476501e8-09a2-5ece-8869-fb82de89a1fa" +version = "0.6.27" + +[[SciMLBase]] +deps = ["ArrayInterface", "CommonSolve", "ConstructionBase", "Distributed", "DocStringExtensions", "IteratorInterfaceExtensions", "LinearAlgebra", "Logging", "RecipesBase", "RecursiveArrayTools", "StaticArrays", "Statistics", "Tables", "TreeViews"] +git-tree-sha1 = "91e29a2bb257a4b992c48f35084064578b87d364" +uuid = "0bca4576-84f4-4d90-8ffe-ffa030f20462" +version = "1.19.0" + +[[Scratch]] +deps = ["Dates"] +git-tree-sha1 = "0b4b7f1393cff97c33891da2a0bf69c6ed241fda" +uuid = "6c6a2e73-6563-6170-7368-637461726353" +version = "1.1.0" + +[[Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" + +[[Setfield]] +deps = ["ConstructionBase", "Future", "MacroTools", "Requires"] +git-tree-sha1 = "def0718ddbabeb5476e51e5a43609bee889f285d" +uuid = "efcf1570-3423-57d1-acb7-fd33fddbac46" +version = "0.8.0" + +[[SharedArrays]] +deps = ["Distributed", "Mmap", "Random", "Serialization"] +uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383" + +[[Showoff]] +deps = ["Dates", "Grisu"] +git-tree-sha1 = "91eddf657aca81df9ae6ceb20b959ae5653ad1de" +uuid = "992d4aef-0814-514b-bc4d-f2e9a6c4116f" +version = "1.0.3" + +[[Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" + +[[SortingAlgorithms]] +deps = ["DataStructures"] +git-tree-sha1 = "b3363d7460f7d098ca0912c69b082f75625d7508" +uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c" +version = "1.0.1" + +[[SparseArrays]] +deps = ["LinearAlgebra", "Random"] +uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" + +[[SpecialFunctions]] +deps = ["ChainRulesCore", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"] +git-tree-sha1 = "ad42c30a6204c74d264692e633133dcea0e8b14e" +uuid = "276daf66-3868-5448-9aa4-cd146d93841b" +version = "1.6.2" + +[[Static]] +deps = ["IfElse"] +git-tree-sha1 = "a8f30abc7c64a39d389680b74e749cf33f872a70" +uuid = "aedffcd0-7271-4cad-89d0-dc628f76c6d3" +version = "0.3.3" + +[[StaticArrays]] +deps = ["LinearAlgebra", "Random", "Statistics"] +git-tree-sha1 = "3240808c6d463ac46f1c1cd7638375cd22abbccb" +uuid = "90137ffa-7385-5640-81b9-e52037218182" +version = "1.2.12" + +[[Statistics]] +deps = ["LinearAlgebra", "SparseArrays"] +uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" + +[[StatsAPI]] +git-tree-sha1 = "1958272568dc176a1d881acb797beb909c785510" +uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0" +version = "1.0.0" + +[[StatsBase]] +deps = ["DataAPI", "DataStructures", "LinearAlgebra", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"] +git-tree-sha1 = "8cbbc098554648c84f79a463c9ff0fd277144b6c" +uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" +version = "0.33.10" + +[[StatsFuns]] +deps = ["ChainRulesCore", "IrrationalConstants", "LogExpFunctions", "Reexport", "Rmath", "SpecialFunctions"] +git-tree-sha1 = "46d7ccc7104860c38b11966dd1f72ff042f382e4" +uuid = "4c63d2b9-4356-54db-8cca-17b64c39e42c" +version = "0.9.10" + +[[StrideArraysCore]] +deps = ["ArrayInterface", "CloseOpenIntervals", "IfElse", "LayoutPointers", "ManualMemory", "Requires", "SIMDTypes", "Static", "ThreadingUtilities"] +git-tree-sha1 = "1258e25e171aec339866f283a11e7d75867e77d7" +uuid = "7792a7ef-975c-4747-a70f-980b88e8d1da" +version = "0.2.4" + +[[StructArrays]] +deps = ["Adapt", "DataAPI", "StaticArrays", "Tables"] +git-tree-sha1 = "2ce41e0d042c60ecd131e9fb7154a3bfadbf50d3" +uuid = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" +version = "0.6.3" + +[[SuiteSparse]] +deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"] +uuid = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9" + +[[TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" + +[[TableTraits]] +deps = ["IteratorInterfaceExtensions"] +git-tree-sha1 = "c06b2f539df1c6efa794486abfb6ed2022561a39" +uuid = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c" +version = "1.0.1" + +[[Tables]] +deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "LinearAlgebra", "TableTraits", "Test"] +git-tree-sha1 = "1162ce4a6c4b7e31e0e6b14486a6986951c73be9" +uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" +version = "1.5.2" + +[[Tar]] +deps = ["ArgTools", "SHA"] +uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" + +[[TensorCore]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "1feb45f88d133a655e001435632f019a9a1bcdb6" +uuid = "62fd8b95-f654-4bbd-a8a5-9c27f68ccd50" +version = "0.1.1" + +[[Test]] +deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[[ThreadingUtilities]] +deps = ["ManualMemory"] +git-tree-sha1 = "03013c6ae7f1824131b2ae2fc1d49793b51e8394" +uuid = "8290d209-cae3-49c0-8002-c8c24d57dab5" +version = "0.4.6" + +[[TreeViews]] +deps = ["Test"] +git-tree-sha1 = "8d0d7a3fe2f30d6a7f833a5f19f7c7a5b396eae6" +uuid = "a2a6695c-b41b-5b7d-aed9-dbfdeacea5d7" +version = "0.3.0" + +[[TriangularSolve]] +deps = ["CloseOpenIntervals", "IfElse", "LayoutPointers", "LinearAlgebra", "LoopVectorization", "Polyester", "Static", "VectorizationBase"] +git-tree-sha1 = "ed55426a514db35f58d36c3812aae89cfc057401" +uuid = "d5829a12-d9aa-46ab-831f-fb7c9ab06edf" +version = "0.1.6" + +[[URIs]] +git-tree-sha1 = "97bbe755a53fe859669cd907f2d96aee8d2c1355" +uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" +version = "1.3.0" + +[[UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" + +[[UnPack]] +git-tree-sha1 = "387c1f73762231e86e0c9c5443ce3b4a0a9a0c2b" +uuid = "3a884ed6-31ef-47d7-9d2a-63182c4928ed" +version = "1.0.2" + +[[Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" + +[[VectorizationBase]] +deps = ["ArrayInterface", "CPUSummary", "HostCPUFeatures", "Hwloc", "IfElse", "LayoutPointers", "Libdl", "LinearAlgebra", "SIMDTypes", "Static"] +git-tree-sha1 = "3e2385f4ec895e694c24a1d5aba58cb6d27cf8b6" +uuid = "3d5dd08c-fd9d-11e8-17fa-ed2836048c2f" +version = "0.21.10" + +[[Wayland_jll]] +deps = ["Artifacts", "Expat_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Pkg", "XML2_jll"] +git-tree-sha1 = "3e61f0b86f90dacb0bc0e73a0c5a83f6a8636e23" +uuid = "a2964d1f-97da-50d4-b82a-358c7fce9d89" +version = "1.19.0+0" + +[[Wayland_protocols_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Wayland_jll"] +git-tree-sha1 = "2839f1c1296940218e35df0bbb220f2a79686670" +uuid = "2381bf8a-dfd0-557d-9999-79630e7b1b91" +version = "1.18.0+4" + +[[XML2_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Libiconv_jll", "Pkg", "Zlib_jll"] +git-tree-sha1 = "1acf5bdf07aa0907e0a37d3718bb88d4b687b74a" +uuid = "02c8fc9c-b97f-50b9-bbe4-9be30ff0a78a" +version = "2.9.12+0" + +[[XSLT_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Libgcrypt_jll", "Libgpg_error_jll", "Libiconv_jll", "Pkg", "XML2_jll", "Zlib_jll"] +git-tree-sha1 = "91844873c4085240b95e795f692c4cec4d805f8a" +uuid = "aed1982a-8fda-507f-9586-7b0439959a61" +version = "1.1.34+0" + +[[Xorg_libX11_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libxcb_jll", "Xorg_xtrans_jll"] +git-tree-sha1 = "5be649d550f3f4b95308bf0183b82e2582876527" +uuid = "4f6342f7-b3d2-589e-9d20-edeb45f2b2bc" +version = "1.6.9+4" + +[[Xorg_libXau_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "4e490d5c960c314f33885790ed410ff3a94ce67e" +uuid = "0c0b7dd1-d40b-584c-a123-a41640f87eec" +version = "1.0.9+4" + +[[Xorg_libXcursor_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libXfixes_jll", "Xorg_libXrender_jll"] +git-tree-sha1 = "12e0eb3bc634fa2080c1c37fccf56f7c22989afd" +uuid = "935fb764-8cf2-53bf-bb30-45bb1f8bf724" +version = "1.2.0+4" + +[[Xorg_libXdmcp_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "4fe47bd2247248125c428978740e18a681372dd4" +uuid = "a3789734-cfe1-5b06-b2d0-1dd0d9d62d05" +version = "1.1.3+4" + +[[Xorg_libXext_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll"] +git-tree-sha1 = "b7c0aa8c376b31e4852b360222848637f481f8c3" +uuid = "1082639a-0dae-5f34-9b06-72781eeb8cb3" +version = "1.3.4+4" + +[[Xorg_libXfixes_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll"] +git-tree-sha1 = "0e0dc7431e7a0587559f9294aeec269471c991a4" +uuid = "d091e8ba-531a-589c-9de9-94069b037ed8" +version = "5.0.3+4" + +[[Xorg_libXi_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libXext_jll", "Xorg_libXfixes_jll"] +git-tree-sha1 = "89b52bc2160aadc84d707093930ef0bffa641246" +uuid = "a51aa0fd-4e3c-5386-b890-e753decda492" +version = "1.7.10+4" + +[[Xorg_libXinerama_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libXext_jll"] +git-tree-sha1 = "26be8b1c342929259317d8b9f7b53bf2bb73b123" +uuid = "d1454406-59df-5ea1-beac-c340f2130bc3" +version = "1.1.4+4" + +[[Xorg_libXrandr_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libXext_jll", "Xorg_libXrender_jll"] +git-tree-sha1 = "34cea83cb726fb58f325887bf0612c6b3fb17631" +uuid = "ec84b674-ba8e-5d96-8ba1-2a689ba10484" +version = "1.5.2+4" + +[[Xorg_libXrender_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll"] +git-tree-sha1 = "19560f30fd49f4d4efbe7002a1037f8c43d43b96" +uuid = "ea2f1a96-1ddc-540d-b46f-429655e07cfa" +version = "0.9.10+4" + +[[Xorg_libpthread_stubs_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "6783737e45d3c59a4a4c4091f5f88cdcf0908cbb" +uuid = "14d82f49-176c-5ed1-bb49-ad3f5cbd8c74" +version = "0.1.0+3" + +[[Xorg_libxcb_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "XSLT_jll", "Xorg_libXau_jll", "Xorg_libXdmcp_jll", "Xorg_libpthread_stubs_jll"] +git-tree-sha1 = "daf17f441228e7a3833846cd048892861cff16d6" +uuid = "c7cfdc94-dc32-55de-ac96-5a1b8d977c5b" +version = "1.13.0+3" + +[[Xorg_libxkbfile_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll"] +git-tree-sha1 = "926af861744212db0eb001d9e40b5d16292080b2" +uuid = "cc61e674-0454-545c-8b26-ed2c68acab7a" +version = "1.1.0+4" + +[[Xorg_xcb_util_image_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_xcb_util_jll"] +git-tree-sha1 = "0fab0a40349ba1cba2c1da699243396ff8e94b97" +uuid = "12413925-8142-5f55-bb0e-6d7ca50bb09b" +version = "0.4.0+1" + +[[Xorg_xcb_util_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libxcb_jll"] +git-tree-sha1 = "e7fd7b2881fa2eaa72717420894d3938177862d1" +uuid = "2def613f-5ad1-5310-b15b-b15d46f528f5" +version = "0.4.0+1" + +[[Xorg_xcb_util_keysyms_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_xcb_util_jll"] +git-tree-sha1 = "d1151e2c45a544f32441a567d1690e701ec89b00" +uuid = "975044d2-76e6-5fbe-bf08-97ce7c6574c7" +version = "0.4.0+1" + +[[Xorg_xcb_util_renderutil_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_xcb_util_jll"] +git-tree-sha1 = "dfd7a8f38d4613b6a575253b3174dd991ca6183e" +uuid = "0d47668e-0667-5a69-a72c-f761630bfb7e" +version = "0.3.9+1" + +[[Xorg_xcb_util_wm_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_xcb_util_jll"] +git-tree-sha1 = "e78d10aab01a4a154142c5006ed44fd9e8e31b67" +uuid = "c22f9ab0-d5fe-5066-847c-f4bb1cd4e361" +version = "0.4.1+1" + +[[Xorg_xkbcomp_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libxkbfile_jll"] +git-tree-sha1 = "4bcbf660f6c2e714f87e960a171b119d06ee163b" +uuid = "35661453-b289-5fab-8a00-3d9160c6a3a4" +version = "1.4.2+4" + +[[Xorg_xkeyboard_config_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_xkbcomp_jll"] +git-tree-sha1 = "5c8424f8a67c3f2209646d4425f3d415fee5931d" +uuid = "33bec58e-1273-512f-9401-5d533626f822" +version = "2.27.0+4" + +[[Xorg_xtrans_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "79c31e7844f6ecf779705fbc12146eb190b7d845" +uuid = "c5fb5394-a638-5e4d-96e5-b29de1b5cf10" +version = "1.4.0+3" + +[[Zlib_jll]] +deps = ["Libdl"] +uuid = "83775a58-1f1d-513f-b197-d71354ab007a" + +[[Zstd_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "cc4bf3fdde8b7e3e9fa0351bdeedba1cf3b7f6e6" +uuid = "3161d3a3-bdf6-5164-811a-617609db77b4" +version = "1.5.0+0" + +[[Zygote]] +deps = ["AbstractFFTs", "ChainRules", "ChainRulesCore", "DiffRules", "Distributed", "FillArrays", "ForwardDiff", "IRTools", "InteractiveUtils", "LinearAlgebra", "MacroTools", "NaNMath", "Random", "Requires", "SpecialFunctions", "Statistics", "ZygoteRules"] +git-tree-sha1 = "4b799addc63aa77ad4112cede8086564d9068511" +uuid = "e88e6eb3-aa80-5325-afca-941959d7151f" +version = "0.6.22" + +[[ZygoteRules]] +deps = ["MacroTools"] +git-tree-sha1 = "9e7a1e8ca60b742e508a315c17eef5211e7fbfd7" +uuid = "700de1a5-db45-46bc-99cf-38207098b444" +version = "0.2.1" + +[[libass_jll]] +deps = ["Artifacts", "Bzip2_jll", "FreeType2_jll", "FriBidi_jll", "HarfBuzz_jll", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"] +git-tree-sha1 = "5982a94fcba20f02f42ace44b9894ee2b140fe47" +uuid = "0ac62f75-1d6f-5e53-bd7c-93b484bb37c0" +version = "0.15.1+0" + +[[libfdk_aac_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "daacc84a041563f965be61859a36e17c4e4fcd55" +uuid = "f638f0a6-7fb0-5443-88ba-1cc74229b280" +version = "2.0.2+0" + +[[libpng_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"] +git-tree-sha1 = "94d180a6d2b5e55e447e2d27a29ed04fe79eb30c" +uuid = "b53b4c65-9356-5827-b1ea-8c7a1a84506f" +version = "1.6.38+0" + +[[libvorbis_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Ogg_jll", "Pkg"] +git-tree-sha1 = "c45f4e40e7aafe9d086379e5578947ec8b95a8fb" +uuid = "f27f6e37-5d2b-51aa-960f-b287f2bc3b7a" +version = "1.3.7+0" + +[[nghttp2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" + +[[p7zip_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" + +[[x264_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "4fea590b89e6ec504593146bf8b988b2c00922b2" +uuid = "1270edf5-f2f9-52d2-97e9-ab00b5d0237a" +version = "2021.5.5+0" + +[[x265_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "ee567a171cce03570d77ad3a43e90218e38937a9" +uuid = "dfaa095f-4041-5dcd-9319-2fabd8486b76" +version = "3.5.0+0" + +[[xkbcommon_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Wayland_jll", "Wayland_protocols_jll", "Xorg_libxcb_jll", "Xorg_xkeyboard_config_jll"] +git-tree-sha1 = "ece2350174195bb31de1a63bea3a41ae1aa593b6" +uuid = "d8fb68d0-12a3-5cfd-a85a-d49703b185fd" +version = "0.9.1+5" diff --git a/examples/c-comparisons/Project.toml b/examples/c-comparisons/Project.toml new file mode 100644 index 00000000..e2e658da --- /dev/null +++ b/examples/c-comparisons/Project.toml @@ -0,0 +1,11 @@ +[deps] +ApproximateGPs = "298c2ebc-0411-48ad-af38-99e88101b606" +Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" +LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +Literate = "98b081ad-f1c9-55d3-8b20-4c87d4299306" +LogExpFunctions = "2ab3a3ac-af41-5b50-aa03-7779005ae688" +Optim = "429524aa-4258-5aef-a3af-852621145aeb" +ParameterHandling = "2412ca09-6db7-441c-8e3a-88d5709968c5" +Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" diff --git a/examples/c-comparisons/script.jl b/examples/c-comparisons/script.jl new file mode 100644 index 00000000..dc33fe96 --- /dev/null +++ b/examples/c-comparisons/script.jl @@ -0,0 +1,125 @@ +# # Binary Classification with Laplace approximation +# +# This example demonstrates how to carry out non-conjugate Gaussian process +# inference using the Laplace approximation. +# +# For a basic introduction to the functionality of this library, please refer +# to the [User Guide](@ref). +# +# ## Setup + +using ApproximateGPs +using LinearAlgebra +using Distributions +using LogExpFunctions: logistic, softplus, invsoftplus +#using ParameterHandling +using Zygote +using Optim + +using Plots +default(; legend=:outertopright, size=(700, 400)) + +using Random +Random.seed!(1); + +# ## Generate some training data +# +# We create a binary-labelled toy dataset: + +Xgrid = -4:0.1:29 # for visualization +X = range(0, 23.5; length=48) # training inputs +f(x) = 3 * sin(10 + 0.6x) + sin(0.1x) - 1 # latent function +fs = f.(X) # latent function values at training inputs +const invlink = logistic # could use other invlink, e.g. normcdf(f) = cdf(Normal(), f) +ps = invlink.(fs) # probabilities at the training inputs +Y = [rand(Bernoulli(p)) for p in ps] # observations at the training inputs + +function plot_data() + plot(; xlims=extrema(Xgrid), xticks=0:6:24) + plot!(Xgrid, invlink ∘ f; label="true probabilities") + return scatter!(X, Y; label="observations", color=3) +end + +plot_data() + +# ## Creating the latent GP +# +# Here we write a function that creates our latent GP prior, given the +# hyperparameter vector `theta`. Compared to a "vanilla" GP, the `LatentGP` +# requires a function or functor that maps from the latent GP `f` to the +# distribution of observations `y`. This functor is commonly called "the +# likelihood". + +function build_latent_gp(theta) + # `theta` is unconstrained, but kernel variance and lengthscale must be positive: + variance = softplus(theta[1]) + lengthscale = softplus(theta[2]) + + kernel = variance * with_lengthscale(SqExponentialKernel(), lengthscale) + + dist_y_given_f = BernoulliLikelihood() # has logistic invlink by default + # We could also be explicit and define it as a function: + # dist_y_given_f(f) = Bernoulli(invlink(f)) + + jitter = 1e-8 # required for numeric stability [TODO: where to explain this better?] + return LatentGP(GP(kernel), dist_y_given_f, jitter) +end + +# We define a latent GP at our initial hyperparameter values, here with +# variance 1.0 and lengthscale 5.0: + +theta0 = [invsoftplus(1.0), invsoftplus(5.0)] + +lf = build_latent_gp(theta0) + +lf.f.kernel + +# We can now compute the posterior `p(f | y)` under the Laplace approximation: + +f_post = posterior(LaplaceApproximation(), lf(X), Y) + +# Let's plot samples from this approximate posterior: + +function plot_samples!(Xgrid, fpost; samples=100, color=2) + fx = fpost(Xgrid, 1e-8) + fsamples = rand(fx, samples) + plot!(Xgrid, invlink.(fsamples); color, alpha=0.2, label="") + return plot!(Xgrid, invlink.(mean(fx)); color, lw=2, label="posterior fit") +end + +p1 = plot_data() +plot_samples!(Xgrid, f_post) + +# We can improve this fit by optimising the hyperparameters. + +# ## Optimise the parameters +# +# ApproximateGPs provides a convenience function `build_laplace_objective` that +# constructs an objective function for optimising the hyperparameters, based on +# the Laplace approximation to the log marginal likelihood. +# We pass this objective to Optim.jl's LBFGS optimiser: + +objective = build_laplace_objective(build_latent_gp, X, Y) + +training_results = Optim.optimize( + objective, θ -> only(Zygote.gradient(objective, θ)), theta0, LBFGS(); inplace=false +) + +# Now that we have (hopefully) better hyperparameter values, we need to construct a LatentGP prior with these values: + +lf2 = build_latent_gp(training_results.minimizer) + +lf2.f.kernel + +# Finally, we need to obtain the posterior given the observations again: + +f_post2 = posterior(LaplaceApproximation(; f_init=objective.f), lf2(X), Y) + +# `f_init=objective.f` let's the Laplace approximation "warm-start" at the last +# point of the inner-loop Newton optimisation; `objective.f` is a field on the +# `objective` closure. + +# Let's plot samples from the approximate posterior for the optimised hyperparameters: + +p2 = plot_data() +plot_samples!(Xgrid, f_post2) diff --git a/src/ApproximateGPs.jl b/src/ApproximateGPs.jl index 4f255765..4ec40fc8 100644 --- a/src/ApproximateGPs.jl +++ b/src/ApproximateGPs.jl @@ -22,4 +22,13 @@ include("svgp.jl") include("expected_loglik.jl") include("elbo.jl") +using ForwardDiff + +export LaplaceApproximation +export laplace_lml, build_laplace_objective, build_laplace_objective! +export approx_lml # TODO move to AbstractGPs, see https://github.com/JuliaGaussianProcesses/AbstractGPs.jl/issues/221 +export laplace_steps # TODO clean up/discard +include("laplace.jl") +include("ep.jl") + end diff --git a/src/laplace.jl b/src/laplace.jl new file mode 100644 index 00000000..3f564865 --- /dev/null +++ b/src/laplace.jl @@ -0,0 +1,332 @@ +# workaround for https://github.com/JuliaDiff/ChainRulesCore.jl/issues/470 to avoid Zygote dependency +ignore_ad(closure) = closure() +@non_differentiable ignore_ad(closure) + +struct LaplaceCache{ + Tm<:AbstractMatrix,Tv<:AbstractVector,Td<:Diagonal,Tf<:Real,Tc<:Cholesky +} + K::Tm + f::Tv + W::Td + Wsqrt::Td + loglik::Tf + d_loglik::Tv + B_ch::Tc + a::Tv +end + +function _laplace_train_intermediates(dist_y_given_f, ys, K, f) + # Ψ = log p(y|f) + log p(f) + # = loglik + log p(f) + # dΨ/df = d_loglik - K⁻¹ f + # at fhat: d_loglik = K⁻¹ f + + # d²Ψ/df² = d2_loglik - K⁻¹ + # = -W - K⁻¹ + + ll, d_ll, d2_ll = loglik_and_derivs(dist_y_given_f, ys, f) + + W = -Diagonal(d2_ll) + Wsqrt = sqrt(W) + B = I + Wsqrt * K * Wsqrt + B_ch = cholesky(Symmetric(B)) + b = W * f + d_ll + a = b - Wsqrt * (B_ch \ (Wsqrt * K * b)) + + #return (; K, f, W, Wsqrt, loglik=ll, d_loglik=d_ll, B_ch, a) + return LaplaceCache(K, f, W, Wsqrt, ll, d_ll, B_ch, a) +end + +# dist_y_given_f(f) = Bernoulli(logistic(f)) + +function loglik_and_derivs(dist_y_given_f, ys::AbstractVector, f::AbstractVector{<:Real}) + l(_f, _y) = logpdf(dist_y_given_f(_f), _y) + dl(_f, _y) = ForwardDiff.derivative(__f -> l(__f, _y), _f) + d2l(_f, _y) = ForwardDiff.derivative(__f -> dl(__f, _y), _f) + + ls = map(l, f, ys) + d_ll = map(dl, f, ys) + d2_ll = map(d2l, f, ys) + + ll = sum(ls) + return ll, d_ll, d2_ll +end + +function _newton_step(dist_y_given_f, ys, K, f) + cache = _laplace_train_intermediates(dist_y_given_f, ys, K, f) + fnew = K * cache.a + return fnew, cache +end + +function _laplace_lml(f, cache) + # -a' * f / 2 + loglik - sum(log.(diag(B_ch.L))) + return -cache.a' * f / 2 + cache.loglik - sum(log.(diag(cache.B_ch.L))) +end + +function _newton_inner_loop(dist_y_given_f, ys, K; f_init, maxiter, callback=nothing) + @assert maxiter >= 1 + f = f_init + cache = nothing + for i in 1:maxiter + @debug " - Newton iteration $i: f[1:3]=$(f[1:3])" + fnew, cache = _newton_step(dist_y_given_f, ys, K, f) + if !isnothing(callback) + callback(fnew, cache) + end + + if isapprox(f, fnew) + @debug " + converged" + #f = fnew + break # converged + else + f = fnew + end + end + return f, something(cache) +end + +# Currently, we have a separate function that returns only f_opt to simplify frule/rrule +function newton_inner_loop(dist_y_given_f, ys, K; f_init, maxiter, callback=nothing) + f_opt, _ = _newton_inner_loop(dist_y_given_f, ys, K; f_init, maxiter, callback) + return f_opt +end + +function ChainRulesCore.frule( + (Δself, Δdist_y_given_f, Δys, ΔK), + ::typeof(newton_inner_loop), + dist_y_given_f, + ys, + K; + kwargs..., +) + f_opt, cache = _newton_inner_loop(dist_y_given_f, ys, K; kwargs...) + + # f = K grad_log_p_y_given_f(f) + # fdot = Kdot grad_log_p_y_given_f(f) + K grad2_log_p_y_given_f(f) fdot + # fdot (I - K grad2_log_p_y_given_f(f)) = Kdot grad_log_p_y_given_f(f) + # fdot = (I - K grad2_log_p_y_given_f(f))⁻¹ Kdot grad_log_p_y_given_f(f) + # (I - K grad2_log_p_y_given_f(f)) = (I + K W) = (√W)⁻¹ (I + √W K √W) √W = (√W)⁻¹ B √W + # fdot = (√W)⁻¹ B⁻¹ √W Kdot grad_log_p_y_given_f(f) + ∂f_opt = cache.Wsqrt \ (cache.B_ch \ (cache.Wsqrt * (ΔK * cache.d_loglik))) + + @debug "Hit frule" + + return f_opt, ∂f_opt +end + +function ChainRulesCore.rrule(::typeof(newton_inner_loop), dist_y_given_f, ys, K; kwargs...) + @debug "Hit rrule" + f_opt, cache = _newton_inner_loop(dist_y_given_f, ys, K; kwargs...) + + # f = K (∇log p(y|f)) (RW 3.17) + # δf = δK (∇log p(y|f)) + K δ(∇log p(y|f)) + # = δK (∇log p(y|f)) + K ∇(∇log p(y|f)) δf + # δf (I - K ∇∇log p(y|f)) = δK (∇log p(y|f)) + # δf (I + K W) = δK (∇log p(y|f)) + # δf = (I + K W)⁻¹ δK (∇log p(y|f)) (RW 5.24) + # (I + K W) = (√W)⁻¹ (I + √W K √W) √W = (√W)⁻¹ B √W + # δf = (√W)⁻¹ B⁻¹ √W δK (∇log p(y|f)) + + # ∂f_opt = cache.Wsqrt \ (cache.B_ch \ (cache.Wsqrt * (ΔK * cache.d_loglik))) + + # Re<Δf, δf> = Re<Δf, Wsqrt\inv B\inv Wsqrt δK d_loglik> + # = Re + # + # ΔK = Wsqrt' * cache.B_ch' \ Wsqrt' \ Δf_opt * cache.d_loglik' + + function newton_pullback(Δf_opt) + ∂self = NoTangent() + + ∂dist_y_given_f = @not_implemented( + "gradient of Newton's method w.r.t. likelihood parameters" + ) + + ∂ys = @not_implemented("gradient of Newton's method w.r.t. observations") + + # ∂K = df/dK Δf + ∂K = @thunk(cache.Wsqrt * (cache.B_ch \ (cache.Wsqrt \ Δf_opt)) * cache.d_loglik') + + return (∂self, ∂dist_y_given_f, ∂ys, ∂K) + end + + return f_opt, newton_pullback +end + +function laplace_lml(dist_y_given_f, ys, K, f_opt) + cache = _laplace_train_intermediates(dist_y_given_f, ys, K, f_opt) + return _laplace_lml(f_opt, cache) +end + +function laplace_lml( + dist_y_given_f, ys, K; f_init=zeros(length(ys)), maxiter=100, newton_kwargs... +) + f_opt = newton_inner_loop(dist_y_given_f, ys, K; f_init, maxiter, newton_kwargs...) + return laplace_lml(dist_y_given_f, ys, K, f_opt) +end + +function laplace_lml(lfx::LatentFiniteGP, ys; newton_kwargs...) + dist_y_given_f, K, newton_kwargs = _check_laplace_inputs(lfx, ys; newton_kwargs...) + return laplace_lml(dist_y_given_f, ys, K; newton_kwargs...) +end + +function laplace_f_and_lml(lfx::LatentFiniteGP, ys; newton_kwargs...) + dist_y_given_f, K, newton_kwargs = _check_laplace_inputs(lfx, ys; newton_kwargs...) + f_opt = newton_inner_loop(dist_y_given_f, ys, K; newton_kwargs...) + lml = laplace_lml(dist_y_given_f, ys, K, f_opt) + return f_opt, lml +end + +function _check_laplace_inputs( + lfx::LatentFiniteGP, ys; f_init=nothing, maxiter=100, newton_kwargs... +) + fx = lfx.fx + @assert mean(fx) == zero(mean(fx)) # might work with non-zero prior mean but not checked + @assert length(ys) == length(fx) + dist_y_given_f = lfx.lik + K = cov(fx) + if isnothing(f_init) + f_init = mean(fx) + end + return dist_y_given_f, K, (; f_init, maxiter, newton_kwargs...) +end + +function build_laplace_objective!( + f, + build_latent_gp, + xs, + ys; + newton_warmstart=true, + newton_callback=nothing, + newton_maxiter=100, +) + initialize_f = true + + function objective(theta) + lf = build_latent_gp(theta) + lfx = lf(xs) + ignore_ad() do + # Zygote does not like the try/catch within @info etc. + @debug "Hyperparameters: $theta" + if initialize_f + f .= mean(lfx.fx) + end + end + f_opt, lml = laplace_f_and_lml( + lfx, ys; f_init=f, maxiter=newton_maxiter, callback=newton_callback + ) + ignore_ad() do + if newton_warmstart + f .= f_opt + initialize_f = false + end + end + return -lml + end + + return objective +end + +function build_laplace_objective(build_latent_gp, xs, ys; kwargs...) + f = similar(xs, length(xs)) # will be mutated in-place to "warm-start" the Newton steps + return build_laplace_objective!(f, build_latent_gp, xs, ys; kwargs...) +end + +function laplace_f_cov(cache) + # (K⁻¹ + W)⁻¹ + # = (√W⁻¹) (√W⁻¹ (K⁻¹ + W) √W⁻¹)⁻¹ (√W⁻¹) + # = (√W⁻¹) (√W⁻¹ K⁻¹ √W⁻¹ + I)⁻¹ (√W⁻¹) + # ; (I + C⁻¹)⁻¹ = I - (I + C)⁻¹ + # = (√W⁻¹) (I - (I + √W K √W)⁻¹) (√W⁻¹) + # = (√W⁻¹) (I - B⁻¹) (√W⁻¹) + B_ch = cache.B_ch + Wsqrt_inv = inv(cache.Wsqrt) + return Wsqrt_inv * (I - inv(B_ch)) * Wsqrt_inv +end + +function LaplaceResult(fnew, cache) + f = cache.f + f_cov = laplace_f_cov(cache) + q = MvNormal(f, AbstractGPs._symmetric(f_cov)) + lml_approx = _laplace_lml(f, cache) + + return (; fnew, f_cov, q, lml_approx, cache...) +end + +""" + laplace_steps(lfx::LatentFiniteGP, ys; newton_kwargs...) + +For demonstration purposes: returns an array of all the intermediate +approximations of each Newton step. + +If you are only interested in the actual posterior, use +`posterior(::LaplaceApproximation, ...`. + +TODO figure out how to get the `@ref` to work... +""" +function laplace_steps(lfx::LatentFiniteGP, ys; newton_kwargs...) + dist_y_given_f, K, newton_kwargs = _check_laplace_inputs(lfx, ys; newton_kwargs...) + + res_array = [] + + function store_result!(fnew, cache) + return push!(res_array, LaplaceResult(fnew, cache)) + end + + _ = newton_inner_loop(dist_y_given_f, ys, K; newton_kwargs..., callback=store_result!) + + return res_array +end + +struct LaplaceApproximation{Tkw} + newton_kwargs::Tkw +end + +LaplaceApproximation(; newton_kwargs...) = LaplaceApproximation((; newton_kwargs...)) + +function approx_lml(la::LaplaceApproximation, lfx::LatentFiniteGP, ys) + return laplace_lml(lfx, ys; la.newton_kwargs...) +end + +function AbstractGPs.posterior(la::LaplaceApproximation, lfx::LatentFiniteGP, ys) + dist_y_given_f, K, newton_kwargs = _check_laplace_inputs(lfx, ys; la.newton_kwargs...) + _, cache = _newton_inner_loop(dist_y_given_f, ys, K; newton_kwargs...) + # TODO: should we run newton_inner_loop() and _laplace_train_intermediates() explicitly? + f_post = ApproxPosteriorGP(la, lfx.fx, cache) + # TODO: instead of lfx.fx, should we store lfx itself (including lik)? + return f_post +end + +const LaplacePosteriorGP = ApproxPosteriorGP{<:LaplaceApproximation} + +function _laplace_predict_intermediates(cache, prior_at_x, xnew) + k_x_xnew = cov(prior_at_x.f, prior_at_x.x, xnew) + f_mean = mean(prior_at_x.f, xnew) + k_x_xnew' * cache.d_loglik + L = cache.B_ch.L + v = L \ (cache.Wsqrt * k_x_xnew) + return f_mean, v +end + +function StatsBase.mean_and_var(f::LaplacePosteriorGP, x::AbstractVector) + f_mean, v = _laplace_predict_intermediates(f.data, f.prior, x) + f_var = var(f.prior.f, x) - vec(sum(v .^ 2; dims=1)) + return f_mean, f_var +end + +function StatsBase.mean_and_cov(f::LaplacePosteriorGP, x::AbstractVector) + f_mean, v = _laplace_predict_intermediates(f.data, f.prior, x) + f_cov = cov(f.prior.f, x) - v' * v + return f_mean, f_cov +end + +function Statistics.mean(f::LaplacePosteriorGP, x::AbstractVector) + d_loglik = f.data.d_loglik + return mean(f.prior.f, x) + cov(f.prior.f, f.prior.x, x)' * d_loglik +end + +function Statistics.cov(f::LaplacePosteriorGP, x::AbstractVector) + return last(mean_and_cov(f, x)) +end + +function Statistics.var(f::LaplacePosteriorGP, x::AbstractVector) + return last(mean_and_var(f, x)) +end diff --git a/test/Project.toml b/test/Project.toml index 86a470f6..f8c698e4 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -1,18 +1,30 @@ [deps] AbstractGPs = "99985d1d-32ba-4be9-9821-2ec096f28918" ApproximateGPs = "298c2ebc-0411-48ad-af38-99e88101b606" +ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" +ChainRulesTestUtils = "cdddcdb0-9152-4a09-a978-84456f9df70a" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" +FiniteDifferences = "26cc04aa-876d-5657-8c51-4c34ba976000" Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c" IterTools = "c8e1da08-722c-5040-9ed9-7db0dc04731e" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +LogExpFunctions = "2ab3a3ac-af41-5b50-aa03-7779005ae688" +Optim = "429524aa-4258-5aef-a3af-852621145aeb" PDMats = "90014a1f-27ba-587c-ab20-58faa44d9150" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" [compat] AbstractGPs = "0.4, 0.5" ApproximateGPs = "0.1" +ChainRulesCore = "1" +ChainRulesTestUtils = "1.2.3" Distributions = "0.25" +FiniteDifferences = "0.12" Flux = "0.12" IterTools = "1" +LogExpFunctions = "0.3" +Optim = "1" PDMats = "0.11" +Zygote = "0.6" diff --git a/test/laplace.jl b/test/laplace.jl new file mode 100644 index 00000000..3e09eb24 --- /dev/null +++ b/test/laplace.jl @@ -0,0 +1,182 @@ +function generate_data() + Random.seed!(1) + X = range(0, 23.5; length=48) + fs = @. 3 * sin(10 + 0.6X) + sin(0.1X) - 1 + # invlink = normcdf + invlink = logistic + ps = invlink.(fs) + Y = [rand(Bernoulli(p)) for p in ps] + return X, Y +end + +dist_y_given_f(f) = Bernoulli(logistic(f)) + +function build_latent_gp(theta) + variance = softplus(theta[1]) + lengthscale = softplus(theta[2]) + kernel = variance * with_lengthscale(SqExponentialKernel(), lengthscale) + return LatentGP(GP(kernel), dist_y_given_f, 1e-8) +end + +function optimize_elbo( + build_latent_gp, + theta0, + xs, + ys, + optimizer, + optim_options; + newton_warmstart=true, + newton_callback=nothing, +) + objective = build_laplace_objective( + build_latent_gp, xs, ys; newton_warmstart, newton_callback + ) + objective_grad(θ) = only(Zygote.gradient(objective, θ)) + + training_results = Optim.optimize( + objective, objective_grad, theta0, optimizer, optim_options; inplace=false + ) + + lf = build_latent_gp(training_results.minimizer) + f_post = posterior(LaplaceApproximation(; f_init=objective.f), lf(xs), ys) + return f_post, training_results +end + +@testset "Gaussian" begin + # TODO: check for convergence in one step, and agreement with exact GPR + # move to test/equivalences.jl? +end + +@testset "gradients" begin + X, Y = generate_data() + @testset "laplace_lml" begin + Random.seed!(123) + theta0 = rand(2) + function objective(theta) + lf = build_latent_gp(theta) + lml = laplace_lml(lf(X), Y) + return -lml + end + fd_grad = only(FiniteDifferences.grad(central_fdm(5, 1), objective, theta0)) + ad_grad = only(Zygote.gradient(objective, theta0)) + @test ad_grad ≈ fd_grad + end +end + +@testset "chainrule" begin + Random.seed!(54321) + + xs = [0.2, 0.3, 0.7] + ys = [1, 1, 0] + L = randn(3, 3) + + function newton_inner_loop_from_L(dist_y_given_f, ys, L; kwargs...) + K = L'L + return ApproximateGPs.newton_inner_loop(dist_y_given_f, ys, K; kwargs...) + end + + function ChainRulesCore.frule( + (Δself, Δdist_y_given_f, Δys, ΔL), + ::typeof(newton_inner_loop_from_L), + dist_y_given_f, + ys, + L; + kwargs..., + ) + K = L'L + # K̇ = L̇'L + L'L̇ + ΔK = ΔL'L + L'ΔL + return frule( + (Δself, Δdist_y_given_f, Δys, ΔK), + ApproximateGPs.newton_inner_loop, + dist_y_given_f, + ys, + K; + kwargs..., + ) + end + + function ChainRulesCore.rrule( + ::typeof(newton_inner_loop_from_L), dist_y_given_f, ys, L; kwargs... + ) + K = L'L + f_opt, newton_from_K_pullback = rrule( + ApproximateGPs.newton_inner_loop, dist_y_given_f, ys, K; kwargs... + ) + + function newton_from_L_pullback(Δf_opt) + (∂self, ∂dist_y_given_f, ∂ys, ∂K) = newton_from_K_pullback(Δf_opt) + # Re⟨K̄, K̇⟩ = Re⟨K̄, L̇'L + L'L̇⟩ + # = Re⟨K̄, L̇'L⟩ + Re⟨K̄, L'L̇⟩ + # = Re⟨K̄L', L̇'⟩ + Re⟨LK̄, L̇⟩ + # = Re⟨LK̄', L̇⟩ + Re⟨LK̄, L̇⟩ + # = Re⟨LK̄' + LK̄, L̇⟩ + # = Re⟨L̄, L̇⟩ + # L̄ = L(K̄' + K̄) + ∂L = @thunk(L * (∂K' + ∂K)) + + return (∂self, ∂dist_y_given_f, ∂ys, ∂L) + end + + return f_opt, newton_from_L_pullback + end + + fkwargs = (; f_init=zeros(length(ys)), maxiter=100) + test_frule(newton_inner_loop_from_L, dist_y_given_f, ys, L; fkwargs) + test_rrule(newton_inner_loop_from_L, dist_y_given_f, ys, L; fkwargs) +end + +@testset "optimization" begin + X, Y = generate_data() + theta0 = [0.0, 1.0] + + @testset "reference optimum" begin + function objective(theta) + lf = build_latent_gp(theta) + return -laplace_lml(lf(X), Y) + end + + @testset "NelderMead" begin + expected_thetahat = [7.708967951453345, 1.5182348363613536] + + res = Optim.optimize(objective, theta0, NelderMead()) + #@info res + + @test res.minimizer ≈ expected_thetahat + end + + @testset "gradient-based" begin + expected_thetahat = [7.709076337653239, 1.51820292019697] + + objective_grad(θ) = only(Zygote.gradient(objective, θ)) + res = Optim.optimize(objective, objective_grad, theta0, LBFGS(); inplace=false) + #@info res + + @test res.minimizer ≈ expected_thetahat + end + end + + @testset "warmstart vs coldstart" begin + args = (build_latent_gp, theta0, X, Y, LBFGS(), Optim.Options(; iterations=1000)) + + n_newton_coldstart = 0 + count_coldstart!(_, _) = (n_newton_coldstart += 1) + + _, res_cold = optimize_elbo( + args...; newton_warmstart=false, newton_callback=count_coldstart! + ) + #@info "Coldstart:\n$res_cold" + + n_newton_warmstart = 0 + count_warmstart!(_, _) = (n_newton_warmstart += 1) + + _, res_warm = optimize_elbo( + args...; newton_warmstart=true, newton_callback=count_warmstart! + ) + #@info "Warmstart:\n$res_warm" + + @info "Newton steps: $n_newton_coldstart (coldstart) vs $n_newton_warmstart (warmstart)" + @test n_newton_coldstart - n_newton_warmstart > 100 + @test res_cold.minimizer ≈ res_warm.minimizer + end +end diff --git a/test/runtests.jl b/test/runtests.jl index 23301c13..ae6801a0 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -4,9 +4,16 @@ using ApproximateGPs using Flux using IterTools using AbstractGPs +using AbstractGPs: LatentFiniteGP using Distributions +using LogExpFunctions: logistic using LinearAlgebra using PDMats +using Optim +using Zygote +using ChainRulesCore +using ChainRulesTestUtils +using FiniteDifferences const GROUP = get(ENV, "GROUP", "All") const PKGDIR = dirname(dirname(pathof(ApproximateGPs))) @@ -14,19 +21,27 @@ const PKGDIR = dirname(dirname(pathof(ApproximateGPs))) include("test_utils.jl") @testset "ApproximateGPs" begin - include("svgp.jl") - println(" ") - @info "Ran svgp tests" - - include("elbo.jl") - println(" ") - @info "Ran elbo tests" - include("expected_loglik.jl") println(" ") @info "Ran expected_loglik tests" - include("equivalences.jl") - println(" ") - @info "Ran equivalences tests" + @testset "SVGP" begin + include("svgp.jl") + println(" ") + @info "Ran svgp tests" + + include("elbo.jl") + println(" ") + @info "Ran elbo tests" + + include("equivalences.jl") + println(" ") + @info "Ran equivalences tests" + end + + @testset "Laplace" begin + include("laplace.jl") + println(" ") + @info "Ran laplace tests" + end end From e499f4bf7c9c55e52dda6e14b227b73b893c696a Mon Sep 17 00:00:00 2001 From: ST John Date: Fri, 24 Sep 2021 17:00:22 +0300 Subject: [PATCH 02/29] version bump --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 6028e8f7..44ea6bb9 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "ApproximateGPs" uuid = "298c2ebc-0411-48ad-af38-99e88101b606" authors = ["Ross Viljoen "] -version = "0.1.1" +version = "0.1.2" [deps] AbstractGPs = "99985d1d-32ba-4be9-9821-2ec096f28918" From 38e1076f156e9c1a9f4f69fe7ad13e6cf90724e1 Mon Sep 17 00:00:00 2001 From: st-- Date: Sun, 26 Sep 2021 23:06:10 +0300 Subject: [PATCH 03/29] Update examples/c-comparisons/script.jl Co-authored-by: willtebbutt --- examples/c-comparisons/script.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/c-comparisons/script.jl b/examples/c-comparisons/script.jl index dc33fe96..3f4590df 100644 --- a/examples/c-comparisons/script.jl +++ b/examples/c-comparisons/script.jl @@ -12,7 +12,6 @@ using ApproximateGPs using LinearAlgebra using Distributions using LogExpFunctions: logistic, softplus, invsoftplus -#using ParameterHandling using Zygote using Optim From 05d40f1a3c95b71624f814a051e830c8853e604a Mon Sep 17 00:00:00 2001 From: ST John Date: Mon, 27 Sep 2021 09:30:17 +0300 Subject: [PATCH 04/29] update example tomls --- examples/c-comparisons/Manifest.toml | 185 --------------------------- examples/c-comparisons/Project.toml | 1 - 2 files changed, 186 deletions(-) diff --git a/examples/c-comparisons/Manifest.toml b/examples/c-comparisons/Manifest.toml index c596c51e..b729b082 100644 --- a/examples/c-comparisons/Manifest.toml +++ b/examples/c-comparisons/Manifest.toml @@ -24,11 +24,6 @@ path = "../.." uuid = "298c2ebc-0411-48ad-af38-99e88101b606" version = "0.1.1" -[[ArgCheck]] -git-tree-sha1 = "dedbbb2ddb876f899585c4ec4433265e3017215a" -uuid = "dce04be8-c92d-5529-be00-80e4d2c0e197" -version = "2.1.0" - [[ArgTools]] uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" @@ -44,30 +39,12 @@ uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" [[Base64]] uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" -[[Bijectors]] -deps = ["ArgCheck", "ChainRulesCore", "Compat", "Distributions", "Functors", "IrrationalConstants", "LinearAlgebra", "LogExpFunctions", "MappedArrays", "NonlinearSolve", "Random", "Reexport", "Requires", "SparseArrays", "Statistics"] -git-tree-sha1 = "dca5e02c9426b2f8ce86d8e723d0702ff33df234" -uuid = "76274a88-744f-5084-9051-94815aaf08c4" -version = "0.9.8" - -[[BitTwiddlingConvenienceFunctions]] -deps = ["Static"] -git-tree-sha1 = "652aab0fc0d6d4db4cc726425cadf700e9f473f1" -uuid = "62783981-4cbd-42fc-bca8-16325de8dc4b" -version = "0.1.0" - [[Bzip2_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] git-tree-sha1 = "19a35467a82e236ff51bc17a3a44b69ef35185a2" uuid = "6e34b625-4abd-537c-b88f-471c36dfa7a0" version = "1.0.8+0" -[[CPUSummary]] -deps = ["Hwloc", "IfElse", "Static"] -git-tree-sha1 = "ed720e2622820bf584d4ad90e6fcb93d95170b44" -uuid = "2a0fbf3d-bb9c-48f3-b0a9-814d99fd7ab9" -version = "0.1.3" - [[Cairo_jll]] deps = ["Artifacts", "Bzip2_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "JLLWrappers", "LZO_jll", "Libdl", "Pixman_jll", "Pkg", "Xorg_libXext_jll", "Xorg_libXrender_jll", "Zlib_jll", "libpng_jll"] git-tree-sha1 = "f2202b55d816427cd385a9a4f3ffb226bee80f99" @@ -86,12 +63,6 @@ git-tree-sha1 = "bd4afa1fdeec0c8b89dad3c6e92bc6e3b0fec9ce" uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" version = "1.6.0" -[[CloseOpenIntervals]] -deps = ["ArrayInterface", "Static"] -git-tree-sha1 = "ce9c0d07ed6e1a4fecd2df6ace144cbd29ba6f37" -uuid = "fb6a15b2-703c-40df-9091-08a04967cfa9" -version = "0.1.2" - [[ColorSchemes]] deps = ["ColorTypes", "Colors", "FixedPointNumbers", "Random"] git-tree-sha1 = "9995eb3977fbf67b86d0a0a0508e83017ded03f2" @@ -110,11 +81,6 @@ git-tree-sha1 = "417b0ed7b8b838aa6ca0a87aadf1bb9eb111ce40" uuid = "5ae59095-9a9b-59fe-a467-6f913c188581" version = "0.12.8" -[[CommonSolve]] -git-tree-sha1 = "68a0743f578349ada8bc911a5cbd5a2ef6ed6d1f" -uuid = "38540f10-b2f7-11e9-35d8-d573e4eb0ff2" -version = "0.2.0" - [[CommonSubexpressions]] deps = ["MacroTools", "Test"] git-tree-sha1 = "7b8a93dba8af7e3b42fecabf646260105ac373f7" @@ -136,12 +102,6 @@ git-tree-sha1 = "455419f7e328a1a2493cabc6428d79e951349769" uuid = "a33af91c-f02d-484b-be07-31d278c5ca2b" version = "0.1.1" -[[ConstructionBase]] -deps = ["LinearAlgebra"] -git-tree-sha1 = "f74e9d5388b8620b4cee35d4c5a618dd4dc547f4" -uuid = "187b0558-2788-49d3-abe0-74a17ed4e7c9" -version = "1.3.0" - [[Contour]] deps = ["StaticArrays"] git-tree-sha1 = "9f02045d934dc030edad45944ea80dbd1f0ebea7" @@ -293,10 +253,6 @@ git-tree-sha1 = "e2727f02325451f6b24445cd83bfa9aaac19cbe7" uuid = "d9f16b24-f501-4c13-a1f2-28368ffc5196" version = "0.2.5" -[[Future]] -deps = ["Random"] -uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" - [[GLFW_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Libglvnd_jll", "Pkg", "Xorg_libXcursor_jll", "Xorg_libXi_jll", "Xorg_libXinerama_jll", "Xorg_libXrandr_jll"] git-tree-sha1 = "dba1e8614e98949abfa60480b13653813d8f0157" @@ -362,24 +318,6 @@ git-tree-sha1 = "8a954fed8ac097d5be04921d595f741115c1b2ad" uuid = "2e76f6c2-a576-52d4-95c1-20adfe4de566" version = "2.8.1+0" -[[HostCPUFeatures]] -deps = ["BitTwiddlingConvenienceFunctions", "IfElse", "Libdl", "Static"] -git-tree-sha1 = "3169c8b31863f9a409be1d17693751314241e3eb" -uuid = "3e5b6fbb-0976-4d2c-9146-d79de83f2fb0" -version = "0.1.4" - -[[Hwloc]] -deps = ["Hwloc_jll"] -git-tree-sha1 = "92d99146066c5c6888d5a3abc871e6a214388b91" -uuid = "0e44f5e4-bd66-52a0-8798-143a42290a1d" -version = "2.0.0" - -[[Hwloc_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "3395d4d4aeb3c9d31f5929d32760d8baeee88aaf" -uuid = "e33a78d0-f292-5ffc-b300-72abe9b543c8" -version = "2.5.0+0" - [[IOCapture]] deps = ["Logging", "Random"] git-tree-sha1 = "f7be53659ab06ddc986428d3a9dcc95f6fa6705a" @@ -417,12 +355,6 @@ git-tree-sha1 = "05110a2ab1fc5f932622ffea2a003221f4782c18" uuid = "c8e1da08-722c-5040-9ed9-7db0dc04731e" version = "1.3.0" -[[IterativeSolvers]] -deps = ["LinearAlgebra", "Printf", "Random", "RecipesBase", "SparseArrays"] -git-tree-sha1 = "1a8c6237e78b714e901e406c096fc8a65528af7d" -uuid = "42fd0dbc-a981-5370-80f2-aaf504508153" -version = "0.9.1" - [[IteratorInterfaceExtensions]] git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856" uuid = "82899510-4779-5014-852e-03e436cf321d" @@ -481,12 +413,6 @@ git-tree-sha1 = "a4b12a1bd2ebade87891ab7e36fdbce582301a92" uuid = "23fbe1c1-3f47-55db-b15f-69d7ec21a316" version = "0.15.6" -[[LayoutPointers]] -deps = ["ArrayInterface", "LinearAlgebra", "ManualMemory", "SIMDTypes", "Static"] -git-tree-sha1 = "d2bda6aa0b03ce6f141a2dc73d0bcb7070131adc" -uuid = "10f19ff3-798f-405d-979b-55457f8fc047" -version = "0.1.3" - [[LibCURL]] deps = ["LibCURL_jll", "MozillaCACerts_jll"] uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" @@ -579,28 +505,12 @@ version = "0.3.3" [[Logging]] uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" -[[LoopVectorization]] -deps = ["ArrayInterface", "CPUSummary", "CloseOpenIntervals", "DocStringExtensions", "HostCPUFeatures", "IfElse", "LayoutPointers", "LinearAlgebra", "OffsetArrays", "PolyesterWeave", "Requires", "SLEEFPirates", "Static", "ThreadingUtilities", "UnPack", "VectorizationBase"] -git-tree-sha1 = "4804192466f4d370ca19c9957dfb3d919e6ef77e" -uuid = "bdcacae8-1622-11e9-2a5c-532679323890" -version = "0.12.77" - [[MacroTools]] deps = ["Markdown", "Random"] git-tree-sha1 = "5a5bc6bf062f0f95e62d0fe0a2d99699fed82dd9" uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" version = "0.5.8" -[[ManualMemory]] -git-tree-sha1 = "9cb207b18148b2199db259adfa923b45593fe08e" -uuid = "d125e4d3-2237-4719-b19c-fa641b8a4667" -version = "0.1.6" - -[[MappedArrays]] -git-tree-sha1 = "e8b359ef06ec72e8c030463fe02efe5527ee5142" -uuid = "dbb5928d-eab1-5f90-85c2-b9b0edb7c900" -version = "0.4.1" - [[Markdown]] deps = ["Base64"] uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" @@ -646,18 +556,6 @@ version = "0.3.5" [[NetworkOptions]] uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" -[[NonlinearSolve]] -deps = ["ArrayInterface", "FiniteDiff", "ForwardDiff", "IterativeSolvers", "LinearAlgebra", "RecursiveArrayTools", "RecursiveFactorization", "Reexport", "SciMLBase", "Setfield", "StaticArrays", "UnPack"] -git-tree-sha1 = "e9ffc92217b8709e0cf7b8808f6223a4a0936c95" -uuid = "8913a72c-1f9b-4ce2-8d82-65094dcecaec" -version = "0.3.11" - -[[OffsetArrays]] -deps = ["Adapt"] -git-tree-sha1 = "c0e9e582987d36d5a61e650e6e543b9e44d9914b" -uuid = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" -version = "1.10.7" - [[Ogg_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] git-tree-sha1 = "7937eda4681660b4d6aeeecc2f7e1c81c8ee4e2f" @@ -709,12 +607,6 @@ git-tree-sha1 = "4dd403333bcf0909341cfe57ec115152f937d7d8" uuid = "90014a1f-27ba-587c-ab20-58faa44d9150" version = "0.11.1" -[[ParameterHandling]] -deps = ["Bijectors", "ChainRulesCore", "Compat", "IterTools", "LinearAlgebra", "SparseArrays", "Test"] -git-tree-sha1 = "b454231b4559c118fe3522734c71555592445843" -uuid = "2412ca09-6db7-441c-8e3a-88d5709968c5" -version = "0.3.8" - [[Parameters]] deps = ["OrderedCollections", "UnPack"] git-tree-sha1 = "34c0e9ad262e5f7fc75b10a9952ca7692cfc5fbe" @@ -755,18 +647,6 @@ git-tree-sha1 = "457b13497a3ea4deb33d273a6a5ea15c25c0ebd9" uuid = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" version = "1.22.2" -[[Polyester]] -deps = ["ArrayInterface", "BitTwiddlingConvenienceFunctions", "CPUSummary", "IfElse", "ManualMemory", "PolyesterWeave", "Requires", "Static", "StrideArraysCore", "ThreadingUtilities"] -git-tree-sha1 = "74d358e649e0450cb5d3ff54ca7c8d806ed62765" -uuid = "f517fe37-dbe3-4b94-8317-1923a5111588" -version = "0.5.1" - -[[PolyesterWeave]] -deps = ["BitTwiddlingConvenienceFunctions", "CPUSummary", "IfElse", "Static", "ThreadingUtilities"] -git-tree-sha1 = "371a19bb801c1b420b29141750f3a34d6c6634b9" -uuid = "1d0040c9-8b98-4ee7-8388-3f51789ca0ad" -version = "0.1.0" - [[PositiveFactorizations]] deps = ["LinearAlgebra"] git-tree-sha1 = "17275485f373e6673f7e7f97051f703ed5b15b20" @@ -814,18 +694,6 @@ git-tree-sha1 = "7ad0dfa8d03b7bcf8c597f59f5292801730c55b8" uuid = "01d81517-befc-4cb6-b9ec-a95719d0359c" version = "0.4.1" -[[RecursiveArrayTools]] -deps = ["ArrayInterface", "ChainRulesCore", "DocStringExtensions", "LinearAlgebra", "RecipesBase", "Requires", "StaticArrays", "Statistics", "ZygoteRules"] -git-tree-sha1 = "00bede2eb099dcc1ddc3f9ec02180c326b420ee2" -uuid = "731186ca-8d62-57ce-b412-fbd966d074cd" -version = "2.17.2" - -[[RecursiveFactorization]] -deps = ["LinearAlgebra", "LoopVectorization", "Polyester", "StrideArraysCore", "TriangularSolve"] -git-tree-sha1 = "575c18c6b00ce409f75d96fefe33ebe01575457a" -uuid = "f2c3362d-daeb-58d1-803e-2bc74f2840b4" -version = "0.2.4" - [[Reexport]] git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" uuid = "189a3867-3050-52da-a836-e630ba90ab69" @@ -852,23 +720,6 @@ version = "0.3.0+0" [[SHA]] uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" -[[SIMDTypes]] -git-tree-sha1 = "330289636fb8107c5f32088d2741e9fd7a061a5c" -uuid = "94e857df-77ce-4151-89e5-788b33177be4" -version = "0.1.0" - -[[SLEEFPirates]] -deps = ["IfElse", "Static", "VectorizationBase"] -git-tree-sha1 = "2e8150c7d2a14ac68537c7aac25faa6577aff046" -uuid = "476501e8-09a2-5ece-8869-fb82de89a1fa" -version = "0.6.27" - -[[SciMLBase]] -deps = ["ArrayInterface", "CommonSolve", "ConstructionBase", "Distributed", "DocStringExtensions", "IteratorInterfaceExtensions", "LinearAlgebra", "Logging", "RecipesBase", "RecursiveArrayTools", "StaticArrays", "Statistics", "Tables", "TreeViews"] -git-tree-sha1 = "91e29a2bb257a4b992c48f35084064578b87d364" -uuid = "0bca4576-84f4-4d90-8ffe-ffa030f20462" -version = "1.19.0" - [[Scratch]] deps = ["Dates"] git-tree-sha1 = "0b4b7f1393cff97c33891da2a0bf69c6ed241fda" @@ -878,12 +729,6 @@ version = "1.1.0" [[Serialization]] uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" -[[Setfield]] -deps = ["ConstructionBase", "Future", "MacroTools", "Requires"] -git-tree-sha1 = "def0718ddbabeb5476e51e5a43609bee889f285d" -uuid = "efcf1570-3423-57d1-acb7-fd33fddbac46" -version = "0.8.0" - [[SharedArrays]] deps = ["Distributed", "Mmap", "Random", "Serialization"] uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383" @@ -946,12 +791,6 @@ git-tree-sha1 = "46d7ccc7104860c38b11966dd1f72ff042f382e4" uuid = "4c63d2b9-4356-54db-8cca-17b64c39e42c" version = "0.9.10" -[[StrideArraysCore]] -deps = ["ArrayInterface", "CloseOpenIntervals", "IfElse", "LayoutPointers", "ManualMemory", "Requires", "SIMDTypes", "Static", "ThreadingUtilities"] -git-tree-sha1 = "1258e25e171aec339866f283a11e7d75867e77d7" -uuid = "7792a7ef-975c-4747-a70f-980b88e8d1da" -version = "0.2.4" - [[StructArrays]] deps = ["Adapt", "DataAPI", "StaticArrays", "Tables"] git-tree-sha1 = "2ce41e0d042c60ecd131e9fb7154a3bfadbf50d3" @@ -992,24 +831,6 @@ version = "0.1.1" deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" -[[ThreadingUtilities]] -deps = ["ManualMemory"] -git-tree-sha1 = "03013c6ae7f1824131b2ae2fc1d49793b51e8394" -uuid = "8290d209-cae3-49c0-8002-c8c24d57dab5" -version = "0.4.6" - -[[TreeViews]] -deps = ["Test"] -git-tree-sha1 = "8d0d7a3fe2f30d6a7f833a5f19f7c7a5b396eae6" -uuid = "a2a6695c-b41b-5b7d-aed9-dbfdeacea5d7" -version = "0.3.0" - -[[TriangularSolve]] -deps = ["CloseOpenIntervals", "IfElse", "LayoutPointers", "LinearAlgebra", "LoopVectorization", "Polyester", "Static", "VectorizationBase"] -git-tree-sha1 = "ed55426a514db35f58d36c3812aae89cfc057401" -uuid = "d5829a12-d9aa-46ab-831f-fb7c9ab06edf" -version = "0.1.6" - [[URIs]] git-tree-sha1 = "97bbe755a53fe859669cd907f2d96aee8d2c1355" uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" @@ -1027,12 +848,6 @@ version = "1.0.2" [[Unicode]] uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" -[[VectorizationBase]] -deps = ["ArrayInterface", "CPUSummary", "HostCPUFeatures", "Hwloc", "IfElse", "LayoutPointers", "Libdl", "LinearAlgebra", "SIMDTypes", "Static"] -git-tree-sha1 = "3e2385f4ec895e694c24a1d5aba58cb6d27cf8b6" -uuid = "3d5dd08c-fd9d-11e8-17fa-ed2836048c2f" -version = "0.21.10" - [[Wayland_jll]] deps = ["Artifacts", "Expat_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Pkg", "XML2_jll"] git-tree-sha1 = "3e61f0b86f90dacb0bc0e73a0c5a83f6a8636e23" diff --git a/examples/c-comparisons/Project.toml b/examples/c-comparisons/Project.toml index e2e658da..001bff6d 100644 --- a/examples/c-comparisons/Project.toml +++ b/examples/c-comparisons/Project.toml @@ -5,7 +5,6 @@ LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" Literate = "98b081ad-f1c9-55d3-8b20-4c87d4299306" LogExpFunctions = "2ab3a3ac-af41-5b50-aa03-7779005ae688" Optim = "429524aa-4258-5aef-a3af-852621145aeb" -ParameterHandling = "2412ca09-6db7-441c-8e3a-88d5709968c5" Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" From 6a2c7e5f932e4338ca211934740a63c84edac136 Mon Sep 17 00:00:00 2001 From: ST John Date: Mon, 27 Sep 2021 09:38:44 +0300 Subject: [PATCH 05/29] add comments to example --- examples/c-comparisons/script.jl | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/examples/c-comparisons/script.jl b/examples/c-comparisons/script.jl index 3f4590df..25c8f04c 100644 --- a/examples/c-comparisons/script.jl +++ b/examples/c-comparisons/script.jl @@ -77,6 +77,11 @@ lf.f.kernel f_post = posterior(LaplaceApproximation(), lf(X), Y) +# This finds the mode of the posterior (for the given values of the +# hyperparameters) using iterated Newton's method (i.e. solving an optimisation +# problem) and then constructs a Gaussian approximation to the posterior by +# matching the curvature at the mode. + # Let's plot samples from this approximate posterior: function plot_samples!(Xgrid, fpost; samples=100, color=2) @@ -89,7 +94,9 @@ end p1 = plot_data() plot_samples!(Xgrid, f_post) -# We can improve this fit by optimising the hyperparameters. +# We can improve this fit by optimising the hyperparameters. For exact Gaussian +# process regression, the maximization objective is the marginal likelihood. +# Here, we can only optimise an _approximation_ to the marginal likelihood. # ## Optimise the parameters # From 351cbdb1827a5b213f909858f374d0b6ff65ee6c Mon Sep 17 00:00:00 2001 From: ST John Date: Mon, 27 Sep 2021 09:48:11 +0300 Subject: [PATCH 06/29] address review comments --- LICENSE | 2 +- src/ApproximateGPs.jl | 1 - src/laplace.jl | 26 +++++++++++++++++--------- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/LICENSE b/LICENSE index dde3dc88..936e45ee 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ Copyright (c) 2021 -The JuliaGaussianProcess Contributors +The JuliaGaussianProcesses Contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/src/ApproximateGPs.jl b/src/ApproximateGPs.jl index 4ec40fc8..f6434831 100644 --- a/src/ApproximateGPs.jl +++ b/src/ApproximateGPs.jl @@ -29,6 +29,5 @@ export laplace_lml, build_laplace_objective, build_laplace_objective! export approx_lml # TODO move to AbstractGPs, see https://github.com/JuliaGaussianProcesses/AbstractGPs.jl/issues/221 export laplace_steps # TODO clean up/discard include("laplace.jl") -include("ep.jl") end diff --git a/src/laplace.jl b/src/laplace.jl index 3f564865..ed9c6876 100644 --- a/src/laplace.jl +++ b/src/laplace.jl @@ -1,18 +1,19 @@ # workaround for https://github.com/JuliaDiff/ChainRulesCore.jl/issues/470 to avoid Zygote dependency +# will be replaced by https://github.com/JuliaDiff/ChainRulesCore.jl/pull/229 ignore_ad(closure) = closure() @non_differentiable ignore_ad(closure) struct LaplaceCache{ Tm<:AbstractMatrix,Tv<:AbstractVector,Td<:Diagonal,Tf<:Real,Tc<:Cholesky } - K::Tm - f::Tv - W::Td - Wsqrt::Td - loglik::Tf - d_loglik::Tv - B_ch::Tc - a::Tv + K::Tm # kernel matrix + f::Tv # mode of posterior p(f | y) + W::Td # diagonal matrix of ∂²/∂fᵢ² loglik + Wsqrt::Td # sqrt(W) + loglik::Tf # ∑ᵢlog p(yᵢ|fᵢ) + d_loglik::Tv # ∂/∂fᵢloglik + B_ch::Tc # cholesky(I + Wsqrt * K * Wsqrt) + a::Tv # K⁻¹ f end function _laplace_train_intermediates(dist_y_given_f, ys, K, f) @@ -283,6 +284,14 @@ end LaplaceApproximation(; newton_kwargs...) = LaplaceApproximation((; newton_kwargs...)) +""" + approx_lml(la::LaplaceApproximation, lfx::LatentFiniteGP, ys) + +Compute an approximation to the log of the marginal likelihood (also known as +"evidence"), which can be used to optimise the hyperparameters of `lfx`. + +This should become part of the AbstractGPs API (see JuliaGaussianProcesses/AbstractGPs.jl#221). +""" function approx_lml(la::LaplaceApproximation, lfx::LatentFiniteGP, ys) return laplace_lml(lfx, ys; la.newton_kwargs...) end @@ -292,7 +301,6 @@ function AbstractGPs.posterior(la::LaplaceApproximation, lfx::LatentFiniteGP, ys _, cache = _newton_inner_loop(dist_y_given_f, ys, K; newton_kwargs...) # TODO: should we run newton_inner_loop() and _laplace_train_intermediates() explicitly? f_post = ApproxPosteriorGP(la, lfx.fx, cache) - # TODO: instead of lfx.fx, should we store lfx itself (including lik)? return f_post end From 27f421dc60b498761710ef28d2865aa821c8e870 Mon Sep 17 00:00:00 2001 From: ST John Date: Mon, 27 Sep 2021 10:06:53 +0300 Subject: [PATCH 07/29] more docstrings --- src/laplace.jl | 45 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/src/laplace.jl b/src/laplace.jl index ed9c6876..50911612 100644 --- a/src/laplace.jl +++ b/src/laplace.jl @@ -87,6 +87,11 @@ function _newton_inner_loop(dist_y_given_f, ys, K; f_init, maxiter, callback=not end # Currently, we have a separate function that returns only f_opt to simplify frule/rrule +""" + newton_inner_loop(dist_y_given_f, ys, K; f_init, maxiter, callback=nothing) + +Find the mode of `p(f | y)` using Newton's method. +""" function newton_inner_loop(dist_y_given_f, ys, K; f_init, maxiter, callback=nothing) f_opt, _ = _newton_inner_loop(dist_y_given_f, ys, K; f_init, maxiter, callback) return f_opt @@ -165,6 +170,11 @@ function laplace_lml( return laplace_lml(dist_y_given_f, ys, K, f_opt) end +""" + laplace_lml(lfx::LatentFiniteGP, ys; newton_kwargs...) + +Compute the Laplace approximation to the log marginal likelihood. +""" function laplace_lml(lfx::LatentFiniteGP, ys; newton_kwargs...) dist_y_given_f, K, newton_kwargs = _check_laplace_inputs(lfx, ys; newton_kwargs...) return laplace_lml(dist_y_given_f, ys, K; newton_kwargs...) @@ -202,12 +212,12 @@ function build_laplace_objective!( ) initialize_f = true - function objective(theta) - lf = build_latent_gp(theta) + function objective(args...; kwargs...) + lf = build_latent_gp(args...; kwargs...) lfx = lf(xs) ignore_ad() do # Zygote does not like the try/catch within @info etc. - @debug "Hyperparameters: $theta" + @debug "Hyperparameters: $args" if initialize_f f .= mean(lfx.fx) end @@ -227,11 +237,31 @@ function build_laplace_objective!( return objective end +""" + build_laplace_objective(build_latent_gp, xs, ys; kwargs...) + +Construct a closure that computes the minimisation objective for optimising +hyperparameters of the latent GP in the Laplace approximation. The returned +closure passes its arguments to `build_latent_gp`. + +# Keyword arguments + +- `newton_warmstart=true`: (default) begin Newton optimisation at the mode of + the previous call to the objective +- `newton_callback`: called as `newton_callback(fnew, cache)` after each Newton step +- `newton_maxiter=100`: maximum number of Newton steps. +""" function build_laplace_objective(build_latent_gp, xs, ys; kwargs...) f = similar(xs, length(xs)) # will be mutated in-place to "warm-start" the Newton steps return build_laplace_objective!(f, build_latent_gp, xs, ys; kwargs...) end +""" + laplace_f_cov(cache) + +Compute the covariance of `q(f)` from the results of the training computation +that are stored in a `LaplaceCache`. +""" function laplace_f_cov(cache) # (K⁻¹ + W)⁻¹ # = (√W⁻¹) (√W⁻¹ (K⁻¹ + W) √W⁻¹)⁻¹ (√W⁻¹) @@ -262,7 +292,7 @@ approximations of each Newton step. If you are only interested in the actual posterior, use `posterior(::LaplaceApproximation, ...`. -TODO figure out how to get the `@ref` to work... +TODO figure out how to get the `@ref` to work to point to the LaplaceApproximation-specific `posterior` docstring... """ function laplace_steps(lfx::LatentFiniteGP, ys; newton_kwargs...) dist_y_given_f, K, newton_kwargs = _check_laplace_inputs(lfx, ys; newton_kwargs...) @@ -296,6 +326,13 @@ function approx_lml(la::LaplaceApproximation, lfx::LatentFiniteGP, ys) return laplace_lml(lfx, ys; la.newton_kwargs...) end +""" + posterior(la::LaplaceApproximation, lfx::LatentFiniteGP, ys) + +Construct a Gaussian approximation `q(f)` to the posterior `p(f | y)` using the +Laplace approximation. Solves for the mode of the posterior using Newton's +method. +""" function AbstractGPs.posterior(la::LaplaceApproximation, lfx::LatentFiniteGP, ys) dist_y_given_f, K, newton_kwargs = _check_laplace_inputs(lfx, ys; la.newton_kwargs...) _, cache = _newton_inner_loop(dist_y_given_f, ys, K; newton_kwargs...) From 992814c27ed0cbf2a5b284546323418b458a1e32 Mon Sep 17 00:00:00 2001 From: ST John Date: Mon, 27 Sep 2021 11:10:15 +0300 Subject: [PATCH 08/29] remove varargs to get closure-fields working correctly --- src/laplace.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/laplace.jl b/src/laplace.jl index 50911612..2c842111 100644 --- a/src/laplace.jl +++ b/src/laplace.jl @@ -212,12 +212,12 @@ function build_laplace_objective!( ) initialize_f = true - function objective(args...; kwargs...) - lf = build_latent_gp(args...; kwargs...) + function objective(theta) + lf = build_latent_gp(theta) lfx = lf(xs) ignore_ad() do # Zygote does not like the try/catch within @info etc. - @debug "Hyperparameters: $args" + @debug "Objective arguments: $args" if initialize_f f .= mean(lfx.fx) end @@ -242,7 +242,7 @@ end Construct a closure that computes the minimisation objective for optimising hyperparameters of the latent GP in the Laplace approximation. The returned -closure passes its arguments to `build_latent_gp`. +closure passes its single argument to `build_latent_gp`. # Keyword arguments From 91e54bfc153bc04732c19b33562d30dccd03d3a2 Mon Sep 17 00:00:00 2001 From: ST John Date: Tue, 28 Sep 2021 10:09:32 +0300 Subject: [PATCH 09/29] back to varargs --- src/laplace.jl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/laplace.jl b/src/laplace.jl index 2c842111..778bdf69 100644 --- a/src/laplace.jl +++ b/src/laplace.jl @@ -212,8 +212,8 @@ function build_laplace_objective!( ) initialize_f = true - function objective(theta) - lf = build_latent_gp(theta) + function objective(args...) + lf = build_latent_gp(args...) lfx = lf(xs) ignore_ad() do # Zygote does not like the try/catch within @info etc. @@ -242,7 +242,8 @@ end Construct a closure that computes the minimisation objective for optimising hyperparameters of the latent GP in the Laplace approximation. The returned -closure passes its single argument to `build_latent_gp`. +closure passes its arguments to `build_latent_gp`, which must return the +`LatentGP` prior. # Keyword arguments From 58a4bb1c0e4b28122e29cbcfc6933ea8f72fa959 Mon Sep 17 00:00:00 2001 From: ST John Date: Tue, 28 Sep 2021 10:10:02 +0300 Subject: [PATCH 10/29] fix code comments for Literate.jl style --- examples/c-comparisons/Manifest.toml | 46 ++++++++++++++-------------- examples/c-comparisons/script.jl | 6 ++-- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/examples/c-comparisons/Manifest.toml b/examples/c-comparisons/Manifest.toml index b729b082..2e5e8962 100644 --- a/examples/c-comparisons/Manifest.toml +++ b/examples/c-comparisons/Manifest.toml @@ -19,10 +19,10 @@ uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" version = "3.3.1" [[ApproximateGPs]] -deps = ["AbstractGPs", "ChainRulesCore", "Distributions", "FastGaussQuadrature", "FillArrays", "ForwardDiff", "GPLikelihoods", "KLDivergences", "LinearAlgebra", "QuadGK", "Reexport", "SpecialFunctions", "Statistics", "StatsBase"] +deps = ["AbstractGPs", "ChainRulesCore", "Distributions", "FastGaussQuadrature", "FillArrays", "ForwardDiff", "GPLikelihoods", "KLDivergences", "LinearAlgebra", "Reexport", "SpecialFunctions", "Statistics", "StatsBase"] path = "../.." uuid = "298c2ebc-0411-48ad-af38-99e88101b606" -version = "0.1.1" +version = "0.1.2" [[ArgTools]] uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" @@ -53,21 +53,21 @@ version = "1.16.1+0" [[ChainRules]] deps = ["ChainRulesCore", "Compat", "LinearAlgebra", "Random", "Statistics"] -git-tree-sha1 = "d88340ab502af66cfffc821e70ae72f7dbdce645" +git-tree-sha1 = "74c737978316e19e0706737542037c468b21a8d9" uuid = "082447d4-558c-5d27-93f4-14fc19e9eca2" -version = "1.11.5" +version = "1.11.6" [[ChainRulesCore]] deps = ["Compat", "LinearAlgebra", "SparseArrays"] -git-tree-sha1 = "bd4afa1fdeec0c8b89dad3c6e92bc6e3b0fec9ce" +git-tree-sha1 = "e8a30e8019a512e4b6c56ccebc065026624660e8" uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" -version = "1.6.0" +version = "1.7.0" [[ColorSchemes]] deps = ["ColorTypes", "Colors", "FixedPointNumbers", "Random"] -git-tree-sha1 = "9995eb3977fbf67b86d0a0a0508e83017ded03f2" +git-tree-sha1 = "a851fec56cb73cfdf43762999ec72eff5b86882a" uuid = "35d6a980-a343-548e-a6ea-1d62b119f2f4" -version = "3.14.0" +version = "3.15.0" [[ColorTypes]] deps = ["FixedPointNumbers", "Random"] @@ -89,9 +89,9 @@ version = "0.3.0" [[Compat]] deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "SHA", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"] -git-tree-sha1 = "1a90210acd935f222ea19657f143004d2c2a1117" +git-tree-sha1 = "31d0151f5716b655421d9d75b7fa74cc4e744df2" uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" -version = "3.38.0" +version = "3.39.0" [[CompilerSupportLibraries_jll]] deps = ["Artifacts", "Libdl"] @@ -202,9 +202,9 @@ version = "0.4.7" [[FillArrays]] deps = ["LinearAlgebra", "Random", "SparseArrays", "Statistics"] -git-tree-sha1 = "7f6ad1a7f4621b4ab8e554133dade99ebc6e7221" +git-tree-sha1 = "29890dfbc427afa59598b8cfcc10034719bd7744" uuid = "1a297f60-69ca-5386-bcde-b61e274b549b" -version = "0.12.5" +version = "0.12.6" [[FiniteDiff]] deps = ["ArrayInterface", "LinearAlgebra", "Requires", "SparseArrays", "StaticArrays"] @@ -308,9 +308,9 @@ version = "1.0.2" [[HTTP]] deps = ["Base64", "Dates", "IniFile", "Logging", "MbedTLS", "NetworkOptions", "Sockets", "URIs"] -git-tree-sha1 = "60ed5f1643927479f845b0135bb369b031b541fa" +git-tree-sha1 = "24675428ca27678f003414a98c9e473e45fe6a21" uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3" -version = "0.9.14" +version = "0.9.15" [[HarfBuzz_jll]] deps = ["Artifacts", "Cairo_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "Graphite2_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Pkg"] @@ -386,9 +386,9 @@ version = "0.2.1" [[KernelFunctions]] deps = ["ChainRulesCore", "Compat", "CompositionsBase", "Distances", "FillArrays", "Functors", "IrrationalConstants", "LinearAlgebra", "LogExpFunctions", "Random", "Requires", "SpecialFunctions", "StatsBase", "TensorCore", "Test", "ZygoteRules"] -git-tree-sha1 = "3b7fceeab37b650c280eb072ffe2b868b03a5423" +git-tree-sha1 = "88506985f762e63e5cab00d5d90cacf86c05fa7b" uuid = "ec8451be-7e33-11e9-00cf-bbf324bd1392" -version = "0.10.17" +version = "0.10.18" [[LAME_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] @@ -643,9 +643,9 @@ version = "1.0.14" [[Plots]] deps = ["Base64", "Contour", "Dates", "Downloads", "FFMPEG", "FixedPointNumbers", "GR", "GeometryBasics", "JSON", "Latexify", "LinearAlgebra", "Measures", "NaNMath", "PlotThemes", "PlotUtils", "Printf", "REPL", "Random", "RecipesBase", "RecipesPipeline", "Reexport", "Requires", "Scratch", "Showoff", "SparseArrays", "Statistics", "StatsBase", "UUIDs"] -git-tree-sha1 = "457b13497a3ea4deb33d273a6a5ea15c25c0ebd9" +git-tree-sha1 = "cfbd033def161db9494f86c5d18fbf874e09e514" uuid = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" -version = "1.22.2" +version = "1.22.3" [[PositiveFactorizations]] deps = ["LinearAlgebra"] @@ -753,10 +753,10 @@ deps = ["LinearAlgebra", "Random"] uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" [[SpecialFunctions]] -deps = ["ChainRulesCore", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"] -git-tree-sha1 = "ad42c30a6204c74d264692e633133dcea0e8b14e" +deps = ["ChainRulesCore", "IrrationalConstants", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"] +git-tree-sha1 = "793793f1df98e3d7d554b65a107e9c9a6399a6ed" uuid = "276daf66-3868-5448-9aa4-cd146d93841b" -version = "1.6.2" +version = "1.7.0" [[Static]] deps = ["IfElse"] @@ -1010,9 +1010,9 @@ version = "1.5.0+0" [[Zygote]] deps = ["AbstractFFTs", "ChainRules", "ChainRulesCore", "DiffRules", "Distributed", "FillArrays", "ForwardDiff", "IRTools", "InteractiveUtils", "LinearAlgebra", "MacroTools", "NaNMath", "Random", "Requires", "SpecialFunctions", "Statistics", "ZygoteRules"] -git-tree-sha1 = "4b799addc63aa77ad4112cede8086564d9068511" +git-tree-sha1 = "2c18495c33331497f147b21e7c65c3246cdb59cc" uuid = "e88e6eb3-aa80-5325-afca-941959d7151f" -version = "0.6.22" +version = "0.6.23" [[ZygoteRules]] deps = ["MacroTools"] diff --git a/examples/c-comparisons/script.jl b/examples/c-comparisons/script.jl index 25c8f04c..85fda151 100644 --- a/examples/c-comparisons/script.jl +++ b/examples/c-comparisons/script.jl @@ -50,15 +50,15 @@ plot_data() # likelihood". function build_latent_gp(theta) - # `theta` is unconstrained, but kernel variance and lengthscale must be positive: + ## `theta` is unconstrained, but kernel variance and lengthscale must be positive: variance = softplus(theta[1]) lengthscale = softplus(theta[2]) kernel = variance * with_lengthscale(SqExponentialKernel(), lengthscale) dist_y_given_f = BernoulliLikelihood() # has logistic invlink by default - # We could also be explicit and define it as a function: - # dist_y_given_f(f) = Bernoulli(invlink(f)) + ## We could also be explicit and define it as a function: + ## dist_y_given_f(f) = Bernoulli(invlink(f)) jitter = 1e-8 # required for numeric stability [TODO: where to explain this better?] return LatentGP(GP(kernel), dist_y_given_f, jitter) From 9fc55fa603aa4ff7dc1e75331b54c1d8c049dbe3 Mon Sep 17 00:00:00 2001 From: ST John Date: Tue, 28 Sep 2021 10:12:29 +0300 Subject: [PATCH 11/29] remove ignore_ad workaround - now in ChainRulesCore 1.7 --- Project.toml | 2 +- src/laplace.jl | 10 +++------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/Project.toml b/Project.toml index 44ea6bb9..e7f9e1b4 100644 --- a/Project.toml +++ b/Project.toml @@ -20,7 +20,7 @@ StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" [compat] AbstractGPs = "0.3, 0.4, 0.5" -ChainRulesCore = "1" +ChainRulesCore = "1.7" Distributions = "0.25" FastGaussQuadrature = "0.4" FillArrays = "0.12" diff --git a/src/laplace.jl b/src/laplace.jl index 778bdf69..415ac065 100644 --- a/src/laplace.jl +++ b/src/laplace.jl @@ -1,8 +1,3 @@ -# workaround for https://github.com/JuliaDiff/ChainRulesCore.jl/issues/470 to avoid Zygote dependency -# will be replaced by https://github.com/JuliaDiff/ChainRulesCore.jl/pull/229 -ignore_ad(closure) = closure() -@non_differentiable ignore_ad(closure) - struct LaplaceCache{ Tm<:AbstractMatrix,Tv<:AbstractVector,Td<:Diagonal,Tf<:Real,Tc<:Cholesky } @@ -215,9 +210,10 @@ function build_laplace_objective!( function objective(args...) lf = build_latent_gp(args...) lfx = lf(xs) - ignore_ad() do + ignore_derivatives() do # Zygote does not like the try/catch within @info etc. @debug "Objective arguments: $args" + # Zygote does not like in-place assignments either if initialize_f f .= mean(lfx.fx) end @@ -225,7 +221,7 @@ function build_laplace_objective!( f_opt, lml = laplace_f_and_lml( lfx, ys; f_init=f, maxiter=newton_maxiter, callback=newton_callback ) - ignore_ad() do + ignore_derivatives() do if newton_warmstart f .= f_opt initialize_f = false From bad4753af318d14d9ae10f6a32a30ba75dedd039 Mon Sep 17 00:00:00 2001 From: ST John Date: Tue, 28 Sep 2021 10:20:40 +0300 Subject: [PATCH 12/29] use likelihood object --- examples/c-comparisons/script.jl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/examples/c-comparisons/script.jl b/examples/c-comparisons/script.jl index 85fda151..66890544 100644 --- a/examples/c-comparisons/script.jl +++ b/examples/c-comparisons/script.jl @@ -29,9 +29,14 @@ Xgrid = -4:0.1:29 # for visualization X = range(0, 23.5; length=48) # training inputs f(x) = 3 * sin(10 + 0.6x) + sin(0.1x) - 1 # latent function fs = f.(X) # latent function values at training inputs -const invlink = logistic # could use other invlink, e.g. normcdf(f) = cdf(Normal(), f) + +lik = BernoulliLikelihood() # has logistic invlink by default +## could use other invlink, e.g. normcdf(f) = cdf(Normal(), f) + +invlink = lik.invlink ps = invlink.(fs) # probabilities at the training inputs Y = [rand(Bernoulli(p)) for p in ps] # observations at the training inputs +## could do this in one call as `Y = rand(lik(fs))` function plot_data() plot(; xlims=extrema(Xgrid), xticks=0:6:24) From 82e31ddba2cabe8960248b51d9b3b5e3c1121fc7 Mon Sep 17 00:00:00 2001 From: ST John Date: Tue, 28 Sep 2021 10:51:19 +0300 Subject: [PATCH 13/29] clean up example notebook --- examples/c-comparisons/script.jl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/examples/c-comparisons/script.jl b/examples/c-comparisons/script.jl index 66890544..a7eaf571 100644 --- a/examples/c-comparisons/script.jl +++ b/examples/c-comparisons/script.jl @@ -21,7 +21,7 @@ default(; legend=:outertopright, size=(700, 400)) using Random Random.seed!(1); -# ## Generate some training data +# ## Generate training data # # We create a binary-labelled toy dataset: @@ -46,7 +46,7 @@ end plot_data() -# ## Creating the latent GP +# ## Create a latent GP # # Here we write a function that creates our latent GP prior, given the # hyperparameter vector `theta`. Compared to a "vanilla" GP, the `LatentGP` @@ -103,7 +103,7 @@ plot_samples!(Xgrid, f_post) # process regression, the maximization objective is the marginal likelihood. # Here, we can only optimise an _approximation_ to the marginal likelihood. -# ## Optimise the parameters +# ## Optimise the hyperparameters # # ApproximateGPs provides a convenience function `build_laplace_objective` that # constructs an objective function for optimising the hyperparameters, based on @@ -122,13 +122,13 @@ lf2 = build_latent_gp(training_results.minimizer) lf2.f.kernel -# Finally, we need to obtain the posterior given the observations again: +# Finally, we need to construct again the posterior given the observations for the latent GP with optimised hyperparameters: f_post2 = posterior(LaplaceApproximation(; f_init=objective.f), lf2(X), Y) -# `f_init=objective.f` let's the Laplace approximation "warm-start" at the last -# point of the inner-loop Newton optimisation; `objective.f` is a field on the -# `objective` closure. +# By passing `f_init=objective.f` we let the Laplace approximation "warm-start" +# at the last point of the inner-loop Newton optimisation; `objective.f` is a +# field on the `objective` closure. # Let's plot samples from the approximate posterior for the optimised hyperparameters: From 3282c543a67597bf98c0b80e182216dbf84fcc0b Mon Sep 17 00:00:00 2001 From: ST John Date: Tue, 28 Sep 2021 14:00:13 +0300 Subject: [PATCH 14/29] clean up notebook --- examples/c-comparisons/script.jl | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/examples/c-comparisons/script.jl b/examples/c-comparisons/script.jl index a7eaf571..0d540fd0 100644 --- a/examples/c-comparisons/script.jl +++ b/examples/c-comparisons/script.jl @@ -67,7 +67,7 @@ function build_latent_gp(theta) jitter = 1e-8 # required for numeric stability [TODO: where to explain this better?] return LatentGP(GP(kernel), dist_y_given_f, jitter) -end +end; # We define a latent GP at our initial hyperparameter values, here with # variance 1.0 and lengthscale 5.0: @@ -78,7 +78,8 @@ lf = build_latent_gp(theta0) lf.f.kernel -# We can now compute the posterior `p(f | y)` under the Laplace approximation: +# We can now compute the Laplace approximation ``q(f)`` to the true posterior +# ``p(f | y)``: f_post = posterior(LaplaceApproximation(), lf(X), Y) @@ -108,9 +109,10 @@ plot_samples!(Xgrid, f_post) # ApproximateGPs provides a convenience function `build_laplace_objective` that # constructs an objective function for optimising the hyperparameters, based on # the Laplace approximation to the log marginal likelihood. -# We pass this objective to Optim.jl's LBFGS optimiser: -objective = build_laplace_objective(build_latent_gp, X, Y) +objective = build_laplace_objective(build_latent_gp, X, Y); + +# We pass this objective to Optim.jl's LBFGS optimiser: training_results = Optim.optimize( objective, θ -> only(Zygote.gradient(objective, θ)), theta0, LBFGS(); inplace=false @@ -122,7 +124,8 @@ lf2 = build_latent_gp(training_results.minimizer) lf2.f.kernel -# Finally, we need to construct again the posterior given the observations for the latent GP with optimised hyperparameters: +# Finally, we need to construct again the (approximate) posterior given the +# observations for the latent GP with optimised hyperparameters: f_post2 = posterior(LaplaceApproximation(; f_init=objective.f), lf2(X), Y) From d9e9598a8debc4fd650b4f4359904522cd4da5fa Mon Sep 17 00:00:00 2001 From: ST John Date: Tue, 28 Sep 2021 15:54:43 +0300 Subject: [PATCH 15/29] improve documentation --- src/laplace.jl | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/src/laplace.jl b/src/laplace.jl index 415ac065..cd7a5386 100644 --- a/src/laplace.jl +++ b/src/laplace.jl @@ -1,3 +1,9 @@ +# Implementation follows Rasmussen & Williams, Gaussian Processes for Machine +# Learning, the MIT Press, 2006. In the following referred to as 'RW'. +# Online text: +# - http://www.gaussianprocess.org/gpml/chapters/RW3.pdf +# - http://www.gaussianprocess.org/gpml/chapters/RW5.pdf + struct LaplaceCache{ Tm<:AbstractMatrix,Tv<:AbstractVector,Td<:Diagonal,Tf<:Real,Tc<:Cholesky } @@ -22,6 +28,7 @@ function _laplace_train_intermediates(dist_y_given_f, ys, K, f) ll, d_ll, d2_ll = loglik_and_derivs(dist_y_given_f, ys, f) + # inner loop iteration of RW Algorithm 3.1, lines 4-7 W = -Diagonal(d2_ll) Wsqrt = sqrt(W) B = I + Wsqrt * K * Wsqrt @@ -33,8 +40,12 @@ function _laplace_train_intermediates(dist_y_given_f, ys, K, f) return LaplaceCache(K, f, W, Wsqrt, ll, d_ll, B_ch, a) end -# dist_y_given_f(f) = Bernoulli(logistic(f)) +""" + loglik_and_derivs(dist_y_given_f, ys, f) +`dist_y_given_f` must be a scalar function from a Real to a Distribution object. +`ys` and `f` are vectors of observations and latent function values, respectively. +""" function loglik_and_derivs(dist_y_given_f, ys::AbstractVector, f::AbstractVector{<:Real}) l(_f, _y) = logpdf(dist_y_given_f(_f), _y) dl(_f, _y) = ForwardDiff.derivative(__f -> l(__f, _y), _f) @@ -130,8 +141,8 @@ function ChainRulesCore.rrule(::typeof(newton_inner_loop), dist_y_given_f, ys, K # ∂f_opt = cache.Wsqrt \ (cache.B_ch \ (cache.Wsqrt * (ΔK * cache.d_loglik))) - # Re<Δf, δf> = Re<Δf, Wsqrt\inv B\inv Wsqrt δK d_loglik> - # = Re + # Re⟨Δf, δf⟩ = Re⟨Δf, Wsqrt⁻¹ B⁻¹ Wsqrt δK d_loglik⟩ + # = Re⟨Wsqrt' B⁻¹' Wsqrt⁻¹' Δf d_loglik', δK⟩ # # ΔK = Wsqrt' * cache.B_ch' \ Wsqrt' \ Δf_opt * cache.d_loglik' @@ -342,21 +353,21 @@ const LaplacePosteriorGP = ApproxPosteriorGP{<:LaplaceApproximation} function _laplace_predict_intermediates(cache, prior_at_x, xnew) k_x_xnew = cov(prior_at_x.f, prior_at_x.x, xnew) - f_mean = mean(prior_at_x.f, xnew) + k_x_xnew' * cache.d_loglik + f_mean = mean(prior_at_x.f, xnew) + k_x_xnew' * cache.d_loglik # RW (3.21) L = cache.B_ch.L - v = L \ (cache.Wsqrt * k_x_xnew) + v = L \ (cache.Wsqrt * k_x_xnew) # RW (3.29) return f_mean, v end function StatsBase.mean_and_var(f::LaplacePosteriorGP, x::AbstractVector) f_mean, v = _laplace_predict_intermediates(f.data, f.prior, x) - f_var = var(f.prior.f, x) - vec(sum(v .^ 2; dims=1)) + f_var = var(f.prior.f, x) - vec(sum(v .^ 2; dims=1)) # RW (3.29) |> diag return f_mean, f_var end function StatsBase.mean_and_cov(f::LaplacePosteriorGP, x::AbstractVector) f_mean, v = _laplace_predict_intermediates(f.data, f.prior, x) - f_cov = cov(f.prior.f, x) - v' * v + f_cov = cov(f.prior.f, x) - v' * v # RW (3.29) return f_mean, f_cov end From 164c52c1e6c47485f667b6159235ef544647ed45 Mon Sep 17 00:00:00 2001 From: ST John Date: Tue, 28 Sep 2021 21:18:52 +0300 Subject: [PATCH 16/29] reorganize order of laplace.jl, add more comments, add error for derivatives of _newton_inner_loop --- src/laplace.jl | 294 +++++++++++++++++++++++++++---------------------- 1 file changed, 161 insertions(+), 133 deletions(-) diff --git a/src/laplace.jl b/src/laplace.jl index cd7a5386..0610a414 100644 --- a/src/laplace.jl +++ b/src/laplace.jl @@ -4,6 +4,145 @@ # - http://www.gaussianprocess.org/gpml/chapters/RW3.pdf # - http://www.gaussianprocess.org/gpml/chapters/RW5.pdf +struct LaplaceApproximation{Tkw} + newton_kwargs::Tkw +end + +LaplaceApproximation(; newton_kwargs...) = LaplaceApproximation((; newton_kwargs...)) + +""" + approx_lml(la::LaplaceApproximation, lfx::LatentFiniteGP, ys) + +Compute an approximation to the log of the marginal likelihood (also known as +"evidence"), which can be used to optimise the hyperparameters of `lfx`. + +This should become part of the AbstractGPs API (see JuliaGaussianProcesses/AbstractGPs.jl#221). +""" +function approx_lml(la::LaplaceApproximation, lfx::LatentFiniteGP, ys) + return laplace_lml(lfx, ys; la.newton_kwargs...) +end + +""" + posterior(la::LaplaceApproximation, lfx::LatentFiniteGP, ys) + +Construct a Gaussian approximation `q(f)` to the posterior `p(f | y)` using the +Laplace approximation. Solves for the mode of the posterior using Newton's +method. +""" +function AbstractGPs.posterior(la::LaplaceApproximation, lfx::LatentFiniteGP, ys) + dist_y_given_f, K, newton_kwargs = _check_laplace_inputs(lfx, ys; la.newton_kwargs...) + _, cache = _newton_inner_loop(dist_y_given_f, ys, K; newton_kwargs...) + # TODO: should we run newton_inner_loop() and _laplace_train_intermediates() explicitly? + f_post = ApproxPosteriorGP(la, lfx.fx, cache) + return f_post # TODO return LatentGP(f_post, lfx.lik, lfx.fx.Σy) +end + +""" + build_laplace_objective(build_latent_gp, xs, ys; kwargs...) + +Construct a closure that computes the minimisation objective for optimising +hyperparameters of the latent GP in the Laplace approximation. The returned +closure passes its arguments to `build_latent_gp`, which must return the +`LatentGP` prior. + +# Keyword arguments + +- `newton_warmstart=true`: (default) begin Newton optimisation at the mode of + the previous call to the objective +- `newton_callback`: called as `newton_callback(fnew, cache)` after each Newton step +- `newton_maxiter=100`: maximum number of Newton steps. +""" +function build_laplace_objective(build_latent_gp, xs, ys; kwargs...) + f = similar(xs, length(xs)) # will be mutated in-place to "warm-start" the Newton steps + return build_laplace_objective!(f, build_latent_gp, xs, ys; kwargs...) +end + +function build_laplace_objective!( + f, + build_latent_gp, + xs, + ys; + newton_warmstart=true, + newton_callback=nothing, + newton_maxiter=100, +) + initialize_f = true + + function objective(args...) + lf = build_latent_gp(args...) + lfx = lf(xs) + ignore_derivatives() do + # Zygote does not like the try/catch within @info etc. + @debug "Objective arguments: $args" + # Zygote does not like in-place assignments either + if initialize_f + f .= mean(lfx.fx) + end + end + f_opt, lml = laplace_f_and_lml( + lfx, ys; f_init=f, maxiter=newton_maxiter, callback=newton_callback + ) + ignore_derivatives() do + if newton_warmstart + f .= f_opt + initialize_f = false + end + end + return -lml + end + + return objective +end + +""" + laplace_f_and_lml(lfx::LatentFiniteGP, ys; newton_kwargs...) + +Compute the mode of the posterior and the Laplace approximation to the log +marginal likelihood. +""" +function laplace_f_and_lml(lfx::LatentFiniteGP, ys; newton_kwargs...) + dist_y_given_f, K, newton_kwargs = _check_laplace_inputs(lfx, ys; newton_kwargs...) + f_opt = newton_inner_loop(dist_y_given_f, ys, K; newton_kwargs...) + lml = laplace_lml(dist_y_given_f, ys, K, f_opt) + return f_opt, lml +end + +""" + laplace_lml(lfx::LatentFiniteGP, ys; newton_kwargs...) + +Compute the Laplace approximation to the log marginal likelihood. +""" +function laplace_lml(lfx::LatentFiniteGP, ys; newton_kwargs...) + dist_y_given_f, K, newton_kwargs = _check_laplace_inputs(lfx, ys; newton_kwargs...) + return laplace_lml(dist_y_given_f, ys, K; newton_kwargs...) +end + +function laplace_lml( + dist_y_given_f, ys, K; f_init=zeros(length(ys)), maxiter=100, newton_kwargs... +) + f_opt = newton_inner_loop(dist_y_given_f, ys, K; f_init, maxiter, newton_kwargs...) + return laplace_lml(dist_y_given_f, ys, K, f_opt) +end + +function laplace_lml(dist_y_given_f, ys, K, f_opt) + cache = _laplace_train_intermediates(dist_y_given_f, ys, K, f_opt) + return _laplace_lml(f_opt, cache) +end + +function _check_laplace_inputs( + lfx::LatentFiniteGP, ys; f_init=nothing, maxiter=100, newton_kwargs... +) + fx = lfx.fx + @assert mean(fx) == zero(mean(fx)) # might work with non-zero prior mean but not checked + @assert length(ys) == length(fx) + dist_y_given_f = lfx.lik + K = cov(fx) + if isnothing(f_init) + f_init = mean(fx) + end + return dist_y_given_f, K, (; f_init, maxiter, newton_kwargs...) +end + struct LaplaceCache{ Tm<:AbstractMatrix,Tv<:AbstractVector,Td<:Diagonal,Tf<:Real,Tc<:Cholesky } @@ -61,11 +200,13 @@ end function _newton_step(dist_y_given_f, ys, K, f) cache = _laplace_train_intermediates(dist_y_given_f, ys, K, f) + # inner loop iteration of RW Algorithm 3.1, line 8 fnew = K * cache.a return fnew, cache end function _laplace_lml(f, cache) + # inner loop iteration of RW Algorithm 3.1, line 10 # -a' * f / 2 + loglik - sum(log.(diag(B_ch.L))) return -cache.a' * f / 2 + cache.loglik - sum(log.(diag(cache.B_ch.L))) end @@ -92,6 +233,26 @@ function _newton_inner_loop(dist_y_given_f, ys, K; f_init, maxiter, callback=not return f, something(cache) end +function ChainRulesCore.frule(Δargs, ::typeof(_newton_inner_loop), args...; kwargs...) + return _newton_inner_loop_no_gradient() +end +function ChainRulesCore.rrule(::typeof(_newton_inner_loop), args...; kwargs...) + return _newton_inner_loop_no_gradient() +end +function _newton_inner_loop_no_derivative() + # It's important that we don't simply call _newton_inner_loop and pass the + # resulting cache directly to _laplace_lml. This would result in the wrong + # gradients. Instead, in newton_inner_loop we return only f_opt, which will + # have the correct gradients thanks to the custom frule/rrule for + # newton_inner_loop. The f_opt form of laplace_lml will then explicitly + # call _laplace_train_intermediates, which allows the AD system to + # correctly compute the gradients. + return error( + "Do not try to compute the derivatives of _newton_inner_loop directly. " * + "Instead, call newton_inner_loop, which has the correct frule/rrule.", + ) +end + # Currently, we have a separate function that returns only f_opt to simplify frule/rrule """ newton_inner_loop(dist_y_given_f, ys, K; f_init, maxiter, callback=nothing) @@ -164,106 +325,6 @@ function ChainRulesCore.rrule(::typeof(newton_inner_loop), dist_y_given_f, ys, K return f_opt, newton_pullback end -function laplace_lml(dist_y_given_f, ys, K, f_opt) - cache = _laplace_train_intermediates(dist_y_given_f, ys, K, f_opt) - return _laplace_lml(f_opt, cache) -end - -function laplace_lml( - dist_y_given_f, ys, K; f_init=zeros(length(ys)), maxiter=100, newton_kwargs... -) - f_opt = newton_inner_loop(dist_y_given_f, ys, K; f_init, maxiter, newton_kwargs...) - return laplace_lml(dist_y_given_f, ys, K, f_opt) -end - -""" - laplace_lml(lfx::LatentFiniteGP, ys; newton_kwargs...) - -Compute the Laplace approximation to the log marginal likelihood. -""" -function laplace_lml(lfx::LatentFiniteGP, ys; newton_kwargs...) - dist_y_given_f, K, newton_kwargs = _check_laplace_inputs(lfx, ys; newton_kwargs...) - return laplace_lml(dist_y_given_f, ys, K; newton_kwargs...) -end - -function laplace_f_and_lml(lfx::LatentFiniteGP, ys; newton_kwargs...) - dist_y_given_f, K, newton_kwargs = _check_laplace_inputs(lfx, ys; newton_kwargs...) - f_opt = newton_inner_loop(dist_y_given_f, ys, K; newton_kwargs...) - lml = laplace_lml(dist_y_given_f, ys, K, f_opt) - return f_opt, lml -end - -function _check_laplace_inputs( - lfx::LatentFiniteGP, ys; f_init=nothing, maxiter=100, newton_kwargs... -) - fx = lfx.fx - @assert mean(fx) == zero(mean(fx)) # might work with non-zero prior mean but not checked - @assert length(ys) == length(fx) - dist_y_given_f = lfx.lik - K = cov(fx) - if isnothing(f_init) - f_init = mean(fx) - end - return dist_y_given_f, K, (; f_init, maxiter, newton_kwargs...) -end - -function build_laplace_objective!( - f, - build_latent_gp, - xs, - ys; - newton_warmstart=true, - newton_callback=nothing, - newton_maxiter=100, -) - initialize_f = true - - function objective(args...) - lf = build_latent_gp(args...) - lfx = lf(xs) - ignore_derivatives() do - # Zygote does not like the try/catch within @info etc. - @debug "Objective arguments: $args" - # Zygote does not like in-place assignments either - if initialize_f - f .= mean(lfx.fx) - end - end - f_opt, lml = laplace_f_and_lml( - lfx, ys; f_init=f, maxiter=newton_maxiter, callback=newton_callback - ) - ignore_derivatives() do - if newton_warmstart - f .= f_opt - initialize_f = false - end - end - return -lml - end - - return objective -end - -""" - build_laplace_objective(build_latent_gp, xs, ys; kwargs...) - -Construct a closure that computes the minimisation objective for optimising -hyperparameters of the latent GP in the Laplace approximation. The returned -closure passes its arguments to `build_latent_gp`, which must return the -`LatentGP` prior. - -# Keyword arguments - -- `newton_warmstart=true`: (default) begin Newton optimisation at the mode of - the previous call to the objective -- `newton_callback`: called as `newton_callback(fnew, cache)` after each Newton step -- `newton_maxiter=100`: maximum number of Newton steps. -""" -function build_laplace_objective(build_latent_gp, xs, ys; kwargs...) - f = similar(xs, length(xs)) # will be mutated in-place to "warm-start" the Newton steps - return build_laplace_objective!(f, build_latent_gp, xs, ys; kwargs...) -end - """ laplace_f_cov(cache) @@ -316,39 +377,6 @@ function laplace_steps(lfx::LatentFiniteGP, ys; newton_kwargs...) return res_array end -struct LaplaceApproximation{Tkw} - newton_kwargs::Tkw -end - -LaplaceApproximation(; newton_kwargs...) = LaplaceApproximation((; newton_kwargs...)) - -""" - approx_lml(la::LaplaceApproximation, lfx::LatentFiniteGP, ys) - -Compute an approximation to the log of the marginal likelihood (also known as -"evidence"), which can be used to optimise the hyperparameters of `lfx`. - -This should become part of the AbstractGPs API (see JuliaGaussianProcesses/AbstractGPs.jl#221). -""" -function approx_lml(la::LaplaceApproximation, lfx::LatentFiniteGP, ys) - return laplace_lml(lfx, ys; la.newton_kwargs...) -end - -""" - posterior(la::LaplaceApproximation, lfx::LatentFiniteGP, ys) - -Construct a Gaussian approximation `q(f)` to the posterior `p(f | y)` using the -Laplace approximation. Solves for the mode of the posterior using Newton's -method. -""" -function AbstractGPs.posterior(la::LaplaceApproximation, lfx::LatentFiniteGP, ys) - dist_y_given_f, K, newton_kwargs = _check_laplace_inputs(lfx, ys; la.newton_kwargs...) - _, cache = _newton_inner_loop(dist_y_given_f, ys, K; newton_kwargs...) - # TODO: should we run newton_inner_loop() and _laplace_train_intermediates() explicitly? - f_post = ApproxPosteriorGP(la, lfx.fx, cache) - return f_post -end - const LaplacePosteriorGP = ApproxPosteriorGP{<:LaplaceApproximation} function _laplace_predict_intermediates(cache, prior_at_x, xnew) From c1b47d3c9d573c953cc0e7b80ac397dff737502e Mon Sep 17 00:00:00 2001 From: ST John Date: Tue, 28 Sep 2021 21:23:11 +0300 Subject: [PATCH 17/29] moved todo into https://github.com/JuliaGaussianProcesses/ApproximateGPs.jl/issues/61 --- examples/c-comparisons/script.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/c-comparisons/script.jl b/examples/c-comparisons/script.jl index 0d540fd0..6e6bc465 100644 --- a/examples/c-comparisons/script.jl +++ b/examples/c-comparisons/script.jl @@ -65,7 +65,7 @@ function build_latent_gp(theta) ## We could also be explicit and define it as a function: ## dist_y_given_f(f) = Bernoulli(invlink(f)) - jitter = 1e-8 # required for numeric stability [TODO: where to explain this better?] + jitter = 1e-8 # required for numeric stability return LatentGP(GP(kernel), dist_y_given_f, jitter) end; From 4ac831390ab22114e34d58c69d47d7d6d74dc3ab Mon Sep 17 00:00:00 2001 From: ST John Date: Tue, 28 Sep 2021 21:24:27 +0300 Subject: [PATCH 18/29] clean up exports --- src/ApproximateGPs.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ApproximateGPs.jl b/src/ApproximateGPs.jl index f6434831..589b423e 100644 --- a/src/ApproximateGPs.jl +++ b/src/ApproximateGPs.jl @@ -25,9 +25,8 @@ include("elbo.jl") using ForwardDiff export LaplaceApproximation -export laplace_lml, build_laplace_objective, build_laplace_objective! +export build_laplace_objective, build_laplace_objective! export approx_lml # TODO move to AbstractGPs, see https://github.com/JuliaGaussianProcesses/AbstractGPs.jl/issues/221 -export laplace_steps # TODO clean up/discard include("laplace.jl") end From 7db3db14b4ac72fcb0cad410450b21c0d2e00241 Mon Sep 17 00:00:00 2001 From: ST John Date: Wed, 29 Sep 2021 10:11:44 +0300 Subject: [PATCH 19/29] fix test --- test/laplace.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/laplace.jl b/test/laplace.jl index 3e09eb24..2f586ff1 100644 --- a/test/laplace.jl +++ b/test/laplace.jl @@ -49,12 +49,12 @@ end @testset "gradients" begin X, Y = generate_data() - @testset "laplace_lml" begin + @testset "approx_lml" begin Random.seed!(123) theta0 = rand(2) function objective(theta) lf = build_latent_gp(theta) - lml = laplace_lml(lf(X), Y) + lml = approx_lml(LaplaceApproximation(), lf(X), Y) return -lml end fd_grad = only(FiniteDifferences.grad(central_fdm(5, 1), objective, theta0)) @@ -133,7 +133,7 @@ end @testset "reference optimum" begin function objective(theta) lf = build_latent_gp(theta) - return -laplace_lml(lf(X), Y) + return -approx_lml(LaplaceApproximation(), lf(X), Y) end @testset "NelderMead" begin From f34c0b04f2fe7fa6e139a927d62cf6a535ce8f32 Mon Sep 17 00:00:00 2001 From: ST John Date: Wed, 29 Sep 2021 10:21:39 +0300 Subject: [PATCH 20/29] add abstractgps internal api test --- src/laplace.jl | 11 +++++++++-- test/laplace.jl | 23 +++++++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/laplace.jl b/src/laplace.jl index 0610a414..cd97e626 100644 --- a/src/laplace.jl +++ b/src/laplace.jl @@ -404,10 +404,17 @@ function Statistics.mean(f::LaplacePosteriorGP, x::AbstractVector) return mean(f.prior.f, x) + cov(f.prior.f, f.prior.x, x)' * d_loglik end +function Statistics.var(f::LaplacePosteriorGP, x::AbstractVector) + return last(mean_and_var(f, x)) +end + function Statistics.cov(f::LaplacePosteriorGP, x::AbstractVector) return last(mean_and_cov(f, x)) end -function Statistics.var(f::LaplacePosteriorGP, x::AbstractVector) - return last(mean_and_var(f, x)) +function Statistics.cov(f::LaplacePosteriorGP, x::AbstractVector, y::AbstractVector) + L = f.data.B_ch.L + vx = L \ (f.data.Wsqrt * cov(f.prior.f, f.prior.x, x)) + vy = L \ (f.data.Wsqrt * cov(f.prior.f, f.prior.x, y)) + return cov(f.prior.f, x, y) - vx' * vy end diff --git a/test/laplace.jl b/test/laplace.jl index 2f586ff1..2a34d923 100644 --- a/test/laplace.jl +++ b/test/laplace.jl @@ -42,6 +42,29 @@ function optimize_elbo( return f_post, training_results end +@testset "AbstractGPs API" begin + rng = MersenneTwister(123456) + N_cond = 5 + N_a = 6 + N_b = 7 + + # Specify prior. + f = GP(Matern32Kernel()) + # Sample from prior. + x = collect(range(-1.0, 1.0; length=N_cond)) + noise_scale = 0.1 + fx = f(x, noise_scale^2) + y = rand(rng, fx) + + jitter = 1e-8 + lf = LatentGP(f, f -> Normal(f, noise_scale), jitter) + f_approx_post = posterior(LaplaceApproximation(), lf(x), y) + + a = collect(range(-1.0, 1.0; length=N_a)) + b = randn(rng, N_b) + AbstractGPs.TestUtils.test_internal_abstractgps_interface(rng, f_approx_post, a, b) +end + @testset "Gaussian" begin # TODO: check for convergence in one step, and agreement with exact GPR # move to test/equivalences.jl? From 90b783b9e9a87c36b961d73665ce8556915c068b Mon Sep 17 00:00:00 2001 From: ST John Date: Wed, 29 Sep 2021 11:15:39 +0300 Subject: [PATCH 21/29] finish tests --- test/laplace.jl | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/test/laplace.jl b/test/laplace.jl index 2a34d923..b0dc1730 100644 --- a/test/laplace.jl +++ b/test/laplace.jl @@ -42,7 +42,7 @@ function optimize_elbo( return f_post, training_results end -@testset "AbstractGPs API" begin +@testset "predictions" begin rng = MersenneTwister(123456) N_cond = 5 N_a = 6 @@ -56,18 +56,28 @@ end fx = f(x, noise_scale^2) y = rand(rng, fx) - jitter = 1e-8 + jitter = 0.0 # not needed in Gaussian case lf = LatentGP(f, f -> Normal(f, noise_scale), jitter) - f_approx_post = posterior(LaplaceApproximation(), lf(x), y) + # in Gaussian case, Laplace converges to f_opt in one step; we need the + # second step to compute the cache at f_opt rather than f_init! + f_approx_post = posterior(LaplaceApproximation(; maxiter=2), lf(x), y) + + @testset "AbstractGPs API" begin + a = collect(range(-1.2, 1.2; length=N_a)) + b = randn(rng, N_b) + AbstractGPs.TestUtils.test_internal_abstractgps_interface(rng, f_approx_post, a, b) + end - a = collect(range(-1.0, 1.0; length=N_a)) - b = randn(rng, N_b) - AbstractGPs.TestUtils.test_internal_abstractgps_interface(rng, f_approx_post, a, b) -end + @testset "equivalence to exact GPR for Gaussian likelihood" begin + f_exact_post = posterior(f(x, noise_scale^2), y) + xt = vcat(x, randn(rng, 3)) # test at training and new points -@testset "Gaussian" begin - # TODO: check for convergence in one step, and agreement with exact GPR - # move to test/equivalences.jl? + m_approx, c_approx = mean_and_cov(f_approx_post(xt)) + m_exact, c_exact = mean_and_cov(f_exact_post(xt)) + + @test m_approx ≈ m_exact + @test c_approx ≈ c_exact + end end @testset "gradients" begin From 68e7145139e64ce8b8ee027678779ee2de149d19 Mon Sep 17 00:00:00 2001 From: st-- Date: Wed, 29 Sep 2021 12:45:18 +0300 Subject: [PATCH 22/29] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théo Galy-Fajou --- examples/c-comparisons/script.jl | 4 ++-- src/laplace.jl | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/examples/c-comparisons/script.jl b/examples/c-comparisons/script.jl index 6e6bc465..011c17fd 100644 --- a/examples/c-comparisons/script.jl +++ b/examples/c-comparisons/script.jl @@ -33,7 +33,7 @@ fs = f.(X) # latent function values at training inputs lik = BernoulliLikelihood() # has logistic invlink by default ## could use other invlink, e.g. normcdf(f) = cdf(Normal(), f) -invlink = lik.invlink +invlink = lik.invlink # logistic function ps = invlink.(fs) # probabilities at the training inputs Y = [rand(Bernoulli(p)) for p in ps] # observations at the training inputs ## could do this in one call as `Y = rand(lik(fs))` @@ -83,7 +83,7 @@ lf.f.kernel f_post = posterior(LaplaceApproximation(), lf(X), Y) -# This finds the mode of the posterior (for the given values of the +# This finds a mode of the posterior (for the given values of the # hyperparameters) using iterated Newton's method (i.e. solving an optimisation # problem) and then constructs a Gaussian approximation to the posterior by # matching the curvature at the mode. diff --git a/src/laplace.jl b/src/laplace.jl index cd97e626..cfff247c 100644 --- a/src/laplace.jl +++ b/src/laplace.jl @@ -282,13 +282,11 @@ function ChainRulesCore.frule( # fdot = (√W)⁻¹ B⁻¹ √W Kdot grad_log_p_y_given_f(f) ∂f_opt = cache.Wsqrt \ (cache.B_ch \ (cache.Wsqrt * (ΔK * cache.d_loglik))) - @debug "Hit frule" return f_opt, ∂f_opt end function ChainRulesCore.rrule(::typeof(newton_inner_loop), dist_y_given_f, ys, K; kwargs...) - @debug "Hit rrule" f_opt, cache = _newton_inner_loop(dist_y_given_f, ys, K; kwargs...) # f = K (∇log p(y|f)) (RW 3.17) From eeb8cee53d0aec0e442da85285ce3a786935fcb7 Mon Sep 17 00:00:00 2001 From: ST John Date: Wed, 29 Sep 2021 13:08:52 +0300 Subject: [PATCH 23/29] bugfix --- src/laplace.jl | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/laplace.jl b/src/laplace.jl index cfff247c..1b97d2ef 100644 --- a/src/laplace.jl +++ b/src/laplace.jl @@ -234,12 +234,12 @@ function _newton_inner_loop(dist_y_given_f, ys, K; f_init, maxiter, callback=not end function ChainRulesCore.frule(Δargs, ::typeof(_newton_inner_loop), args...; kwargs...) - return _newton_inner_loop_no_gradient() + return _newton_inner_loop_no_derivatives() end function ChainRulesCore.rrule(::typeof(_newton_inner_loop), args...; kwargs...) - return _newton_inner_loop_no_gradient() + return _newton_inner_loop_no_derivatives() end -function _newton_inner_loop_no_derivative() +function _newton_inner_loop_no_derivatives() # It's important that we don't simply call _newton_inner_loop and pass the # resulting cache directly to _laplace_lml. This would result in the wrong # gradients. Instead, in newton_inner_loop we return only f_opt, which will @@ -282,7 +282,6 @@ function ChainRulesCore.frule( # fdot = (√W)⁻¹ B⁻¹ √W Kdot grad_log_p_y_given_f(f) ∂f_opt = cache.Wsqrt \ (cache.B_ch \ (cache.Wsqrt * (ΔK * cache.d_loglik))) - return f_opt, ∂f_opt end From 15a886aa92f1835e79edf34238c2d5305d206141 Mon Sep 17 00:00:00 2001 From: ST John Date: Wed, 29 Sep 2021 13:09:13 +0300 Subject: [PATCH 24/29] posterior(::LaplaceApproximation, ...): call newton_inner_loop and _laplace_train_intermediates explicitly --- src/laplace.jl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/laplace.jl b/src/laplace.jl index 1b97d2ef..29e16ba8 100644 --- a/src/laplace.jl +++ b/src/laplace.jl @@ -31,8 +31,11 @@ method. """ function AbstractGPs.posterior(la::LaplaceApproximation, lfx::LatentFiniteGP, ys) dist_y_given_f, K, newton_kwargs = _check_laplace_inputs(lfx, ys; la.newton_kwargs...) - _, cache = _newton_inner_loop(dist_y_given_f, ys, K; newton_kwargs...) - # TODO: should we run newton_inner_loop() and _laplace_train_intermediates() explicitly? + f_opt = newton_inner_loop(dist_y_given_f, ys, K; newton_kwargs...) + # We could call _newton_inner_loop directly and immediately use the + # returned cache, but this would make the posterior call + # non-differentiable, at only a marginal computational saving. + cache = _laplace_train_intermediates(dist_y_given_f, ys, K, f_opt) f_post = ApproxPosteriorGP(la, lfx.fx, cache) return f_post # TODO return LatentGP(f_post, lfx.lik, lfx.fx.Σy) end From c545be009608155408254aea6c5c35d78b129872 Mon Sep 17 00:00:00 2001 From: ST John Date: Wed, 29 Sep 2021 13:09:44 +0300 Subject: [PATCH 25/29] add test for erroring of _newton_inner_loop on backward pass --- test/laplace.jl | 119 ++++++++++++++++++++++++++++-------------------- 1 file changed, 69 insertions(+), 50 deletions(-) diff --git a/test/laplace.jl b/test/laplace.jl index b0dc1730..54589835 100644 --- a/test/laplace.jl +++ b/test/laplace.jl @@ -81,8 +81,9 @@ end end @testset "gradients" begin - X, Y = generate_data() @testset "approx_lml" begin + X, Y = generate_data() + Random.seed!(123) theta0 = rand(2) function objective(theta) @@ -94,69 +95,87 @@ end ad_grad = only(Zygote.gradient(objective, theta0)) @test ad_grad ≈ fd_grad end -end -@testset "chainrule" begin - Random.seed!(54321) + @testset "_newton_inner_loop derivatives not defined" begin + Random.seed!(54321) + + xs = [0.2, 0.3, 0.7] + ys = [1, 1, 0] + theta0 = 1.234 - xs = [0.2, 0.3, 0.7] - ys = [1, 1, 0] - L = randn(3, 3) + function eval_newton_inner_loop(theta) + k = with_lengthscale(Matern52Kernel(), exp(theta)) + K = kernelmatrix(k, xs) + f, cache = ApproximateGPs._newton_inner_loop(dist_y_given_f, ys, K; f_init=zero(xs), maxiter=100) + return f + end - function newton_inner_loop_from_L(dist_y_given_f, ys, L; kwargs...) - K = L'L - return ApproximateGPs.newton_inner_loop(dist_y_given_f, ys, K; kwargs...) + eval_newton_inner_loop(theta0) # forward pass works + @test_throws ErrorException Zygote.gradient(eval_newton_inner_loop, theta0) end - function ChainRulesCore.frule( - (Δself, Δdist_y_given_f, Δys, ΔL), - ::typeof(newton_inner_loop_from_L), - dist_y_given_f, - ys, - L; - kwargs..., - ) - K = L'L - # K̇ = L̇'L + L'L̇ - ΔK = ΔL'L + L'ΔL - return frule( - (Δself, Δdist_y_given_f, Δys, ΔK), - ApproximateGPs.newton_inner_loop, + @testset "newton_inner_loop chain rules" begin + Random.seed!(54321) + + xs = [0.2, 0.3, 0.7] + ys = [1, 1, 0] + L = randn(3, 3) + + function newton_inner_loop_from_L(dist_y_given_f, ys, L; kwargs...) + K = L'L + return ApproximateGPs.newton_inner_loop(dist_y_given_f, ys, K; kwargs...) + end + + function ChainRulesCore.frule( + (Δself, Δdist_y_given_f, Δys, ΔL), + ::typeof(newton_inner_loop_from_L), dist_y_given_f, ys, - K; + L; kwargs..., ) - end + K = L'L + # K̇ = L̇'L + L'L̇ + ΔK = ΔL'L + L'ΔL + return frule( + (Δself, Δdist_y_given_f, Δys, ΔK), + ApproximateGPs.newton_inner_loop, + dist_y_given_f, + ys, + K; + kwargs..., + ) + end - function ChainRulesCore.rrule( - ::typeof(newton_inner_loop_from_L), dist_y_given_f, ys, L; kwargs... - ) - K = L'L - f_opt, newton_from_K_pullback = rrule( - ApproximateGPs.newton_inner_loop, dist_y_given_f, ys, K; kwargs... + function ChainRulesCore.rrule( + ::typeof(newton_inner_loop_from_L), dist_y_given_f, ys, L; kwargs... ) - - function newton_from_L_pullback(Δf_opt) - (∂self, ∂dist_y_given_f, ∂ys, ∂K) = newton_from_K_pullback(Δf_opt) - # Re⟨K̄, K̇⟩ = Re⟨K̄, L̇'L + L'L̇⟩ - # = Re⟨K̄, L̇'L⟩ + Re⟨K̄, L'L̇⟩ - # = Re⟨K̄L', L̇'⟩ + Re⟨LK̄, L̇⟩ - # = Re⟨LK̄', L̇⟩ + Re⟨LK̄, L̇⟩ - # = Re⟨LK̄' + LK̄, L̇⟩ - # = Re⟨L̄, L̇⟩ - # L̄ = L(K̄' + K̄) - ∂L = @thunk(L * (∂K' + ∂K)) - - return (∂self, ∂dist_y_given_f, ∂ys, ∂L) + K = L'L + f_opt, newton_from_K_pullback = rrule( + ApproximateGPs.newton_inner_loop, dist_y_given_f, ys, K; kwargs... + ) + + function newton_from_L_pullback(Δf_opt) + (∂self, ∂dist_y_given_f, ∂ys, ∂K) = newton_from_K_pullback(Δf_opt) + # Re⟨K̄, K̇⟩ = Re⟨K̄, L̇'L + L'L̇⟩ + # = Re⟨K̄, L̇'L⟩ + Re⟨K̄, L'L̇⟩ + # = Re⟨K̄L', L̇'⟩ + Re⟨LK̄, L̇⟩ + # = Re⟨LK̄', L̇⟩ + Re⟨LK̄, L̇⟩ + # = Re⟨LK̄' + LK̄, L̇⟩ + # = Re⟨L̄, L̇⟩ + # L̄ = L(K̄' + K̄) + ∂L = @thunk(L * (∂K' + ∂K)) + + return (∂self, ∂dist_y_given_f, ∂ys, ∂L) + end + + return f_opt, newton_from_L_pullback end - return f_opt, newton_from_L_pullback + fkwargs = (; f_init=zeros(length(ys)), maxiter=100) + test_frule(newton_inner_loop_from_L, dist_y_given_f, ys, L; fkwargs) + test_rrule(newton_inner_loop_from_L, dist_y_given_f, ys, L; fkwargs) end - - fkwargs = (; f_init=zeros(length(ys)), maxiter=100) - test_frule(newton_inner_loop_from_L, dist_y_given_f, ys, L; fkwargs) - test_rrule(newton_inner_loop_from_L, dist_y_given_f, ys, L; fkwargs) end @testset "optimization" begin From 5b8da872a0cc9ac8ba709244fdd2ff398ed07c5c Mon Sep 17 00:00:00 2001 From: st-- Date: Wed, 29 Sep 2021 13:24:23 +0300 Subject: [PATCH 26/29] Apply suggestions from code review Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- test/laplace.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/laplace.jl b/test/laplace.jl index 54589835..06666a2b 100644 --- a/test/laplace.jl +++ b/test/laplace.jl @@ -106,7 +106,9 @@ end function eval_newton_inner_loop(theta) k = with_lengthscale(Matern52Kernel(), exp(theta)) K = kernelmatrix(k, xs) - f, cache = ApproximateGPs._newton_inner_loop(dist_y_given_f, ys, K; f_init=zero(xs), maxiter=100) + f, cache = ApproximateGPs._newton_inner_loop( + dist_y_given_f, ys, K; f_init=zero(xs), maxiter=100 + ) return f end From b3d827ff1dc8f6df21f5b0b63df0eb2b3c057278 Mon Sep 17 00:00:00 2001 From: ST John Date: Wed, 29 Sep 2021 13:38:34 +0300 Subject: [PATCH 27/29] minor cleanup --- src/laplace.jl | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/src/laplace.jl b/src/laplace.jl index 29e16ba8..f3241663 100644 --- a/src/laplace.jl +++ b/src/laplace.jl @@ -10,23 +10,11 @@ end LaplaceApproximation(; newton_kwargs...) = LaplaceApproximation((; newton_kwargs...)) -""" - approx_lml(la::LaplaceApproximation, lfx::LatentFiniteGP, ys) - -Compute an approximation to the log of the marginal likelihood (also known as -"evidence"), which can be used to optimise the hyperparameters of `lfx`. - -This should become part of the AbstractGPs API (see JuliaGaussianProcesses/AbstractGPs.jl#221). -""" -function approx_lml(la::LaplaceApproximation, lfx::LatentFiniteGP, ys) - return laplace_lml(lfx, ys; la.newton_kwargs...) -end - """ posterior(la::LaplaceApproximation, lfx::LatentFiniteGP, ys) Construct a Gaussian approximation `q(f)` to the posterior `p(f | y)` using the -Laplace approximation. Solves for the mode of the posterior using Newton's +Laplace approximation. Solves for a mode of the posterior using Newton's method. """ function AbstractGPs.posterior(la::LaplaceApproximation, lfx::LatentFiniteGP, ys) @@ -40,6 +28,18 @@ function AbstractGPs.posterior(la::LaplaceApproximation, lfx::LatentFiniteGP, ys return f_post # TODO return LatentGP(f_post, lfx.lik, lfx.fx.Σy) end +""" + approx_lml(la::LaplaceApproximation, lfx::LatentFiniteGP, ys) + +Compute an approximation to the log of the marginal likelihood (also known as +"evidence"), which can be used to optimise the hyperparameters of `lfx`. + +This should become part of the AbstractGPs API (see JuliaGaussianProcesses/AbstractGPs.jl#221). +""" +function approx_lml(la::LaplaceApproximation, lfx::LatentFiniteGP, ys) + return laplace_lml(lfx, ys; la.newton_kwargs...) +end + """ build_laplace_objective(build_latent_gp, xs, ys; kwargs...) @@ -56,6 +56,7 @@ closure passes its arguments to `build_latent_gp`, which must return the - `newton_maxiter=100`: maximum number of Newton steps. """ function build_laplace_objective(build_latent_gp, xs, ys; kwargs...) + # TODO assumes type of `xs` will be same as `mean(lfx.fx)` f = similar(xs, length(xs)) # will be mutated in-place to "warm-start" the Newton steps return build_laplace_objective!(f, build_latent_gp, xs, ys; kwargs...) end @@ -121,7 +122,7 @@ function laplace_lml(lfx::LatentFiniteGP, ys; newton_kwargs...) end function laplace_lml( - dist_y_given_f, ys, K; f_init=zeros(length(ys)), maxiter=100, newton_kwargs... + dist_y_given_f, ys, K; f_init, maxiter, newton_kwargs... ) f_opt = newton_inner_loop(dist_y_given_f, ys, K; f_init, maxiter, newton_kwargs...) return laplace_lml(dist_y_given_f, ys, K, f_opt) @@ -137,7 +138,7 @@ function _check_laplace_inputs( ) fx = lfx.fx @assert mean(fx) == zero(mean(fx)) # might work with non-zero prior mean but not checked - @assert length(ys) == length(fx) + @assert length(ys) == length(fx) # LaplaceApproximation currently does not support multi-latent likelihoods dist_y_given_f = lfx.lik K = cov(fx) if isnothing(f_init) From 0b65d1610657b0b5fe301e9c210f2a3eb2db1deb Mon Sep 17 00:00:00 2001 From: ST John Date: Wed, 29 Sep 2021 13:44:18 +0300 Subject: [PATCH 28/29] the -> a mode --- src/laplace.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/laplace.jl b/src/laplace.jl index f3241663..4966c957 100644 --- a/src/laplace.jl +++ b/src/laplace.jl @@ -101,7 +101,7 @@ end """ laplace_f_and_lml(lfx::LatentFiniteGP, ys; newton_kwargs...) -Compute the mode of the posterior and the Laplace approximation to the log +Compute a mode of the posterior and the Laplace approximation to the log marginal likelihood. """ function laplace_f_and_lml(lfx::LatentFiniteGP, ys; newton_kwargs...) @@ -261,7 +261,7 @@ end """ newton_inner_loop(dist_y_given_f, ys, K; f_init, maxiter, callback=nothing) -Find the mode of `p(f | y)` using Newton's method. +Find a mode of `p(f | y)` using Newton's method. """ function newton_inner_loop(dist_y_given_f, ys, K; f_init, maxiter, callback=nothing) f_opt, _ = _newton_inner_loop(dist_y_given_f, ys, K; f_init, maxiter, callback) From bdacea7ab996332d6e4ab6376743c7a3a67b1bc3 Mon Sep 17 00:00:00 2001 From: st-- Date: Wed, 29 Sep 2021 14:19:34 +0300 Subject: [PATCH 29/29] Update src/laplace.jl Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- src/laplace.jl | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/laplace.jl b/src/laplace.jl index 4966c957..f28cb3b7 100644 --- a/src/laplace.jl +++ b/src/laplace.jl @@ -121,9 +121,7 @@ function laplace_lml(lfx::LatentFiniteGP, ys; newton_kwargs...) return laplace_lml(dist_y_given_f, ys, K; newton_kwargs...) end -function laplace_lml( - dist_y_given_f, ys, K; f_init, maxiter, newton_kwargs... -) +function laplace_lml(dist_y_given_f, ys, K; f_init, maxiter, newton_kwargs...) f_opt = newton_inner_loop(dist_y_given_f, ys, K; f_init, maxiter, newton_kwargs...) return laplace_lml(dist_y_given_f, ys, K, f_opt) end