From 8c46be1e7a821271d4146ffaa2b8ad86e8bdd25e Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Tue, 28 Nov 2023 17:00:50 -0300 Subject: [PATCH 1/7] Refactor code to make it more amenable to cross compilation Also cleanup some of it --- src/StaticCompiler.jl | 77 +++++++++++++++------------ src/target.jl | 114 ++++++++++++++++++++++++---------------- test/scripts/interop.jl | 2 +- 3 files changed, 111 insertions(+), 82 deletions(-) diff --git a/src/StaticCompiler.jl b/src/StaticCompiler.jl index 41995f31..6bca325e 100644 --- a/src/StaticCompiler.jl +++ b/src/StaticCompiler.jl @@ -12,9 +12,10 @@ using LLD_jll: lld using StaticTools using StaticTools: @symbolcall, @c_str, println using Core: MethodTable +using Base:BinaryPlatforms.Platform, BinaryPlatforms.HostPlatform, BinaryPlatforms.arch, BinaryPlatforms.os, BinaryPlatforms.libc_str export load_function, compile_shlib, compile_executable -export native_code_llvm, native_code_typed, native_llvm_module, native_code_native +export static_code_llvm, static_code_typed, static_llvm_module, static_code_native export @device_override, @print_and_throw include("interpreter.jl") @@ -32,6 +33,7 @@ compile_executable(f::Function, types::Tuple, path::String, [name::String=string filename::String=name, cflags=``, # Specify libraries you would like to link against, and other compiler options here also_expose=[], + target::StaticTarget=StaticTarget(), method_table=StaticCompiler.method_table, kwargs... ) @@ -96,16 +98,16 @@ Hello, world! ``` """ function compile_executable(f::Function, types=(), path::String="./", name=fix_name(f); - also_expose=Tuple{Function, Tuple{DataType}}[], + also_expose=Tuple{Function, Tuple{DataType}}[], target::StaticTarget=StaticTarget(), kwargs...) - - compile_executable(vcat([(f, types)], also_expose), path, name; kwargs...) + compile_executable(vcat([(f, types)], also_expose), path, name; target, kwargs...) end function compile_executable(funcs::Union{Array,Tuple}, path::String="./", name=fix_name(first(first(funcs))); filename = name, demangle = true, cflags = ``, + target::StaticTarget=StaticTarget(), kwargs... ) @@ -114,12 +116,12 @@ function compile_executable(funcs::Union{Array,Tuple}, path::String="./", name=f isexecutableargtype = tt == Tuple{} || tt == Tuple{Int, Ptr{Ptr{UInt8}}} isexecutableargtype || @warn "input type signature $types should be either `()` or `(Int, Ptr{Ptr{UInt8}})` for standard executables" - rt = last(only(native_code_typed(f, tt; kwargs...))) + rt = last(only(static_code_typed(f, tt; target, kwargs...))) isconcretetype(rt) || error("`$f$types` did not infer to a concrete type. Got `$rt`") nativetype = isprimitivetype(rt) || isa(rt, Ptr) nativetype || @warn "Return type `$rt` of `$f$types` does not appear to be a native type. Consider returning only a single value of a native machine type (i.e., a single float, int/uint, bool, or pointer). \n\nIgnoring this warning may result in Undefined Behavior!" - generate_executable(funcs, path, name, filename; demangle, cflags, kwargs...) + generate_executable(funcs, path, name, filename; demangle, cflags, target, kwargs...) joinpath(abspath(path), filename) end @@ -129,6 +131,7 @@ compile_shlib(f::Function, types::Tuple, [path::String="./"], [name::String=stri filename::String=name, cflags=``, method_table=StaticCompiler.method_table, + target::StaticTarget=StaticTarget(), kwargs...) compile_shlib(funcs::Array, [path::String="./"]; @@ -136,6 +139,7 @@ compile_shlib(funcs::Array, [path::String="./"]; demangle=true, cflags=``, method_table=StaticCompiler.method_table, + target::StaticTarget=StaticTarget(), kwargs...) ``` As `compile_executable`, but compiling to a standalone `.dylib`/`.so` shared library. @@ -169,15 +173,17 @@ julia> ccall(("test", "test.dylib"), Float64, (Int64,), 100_000) """ function compile_shlib(f::Function, types=(), path::String="./", name=fix_name(f); filename=name, + target::StaticTarget=StaticTarget(), kwargs... ) - compile_shlib(((f, types),), path; filename, kwargs...) + compile_shlib(((f, types),), path; filename, target, kwargs...) end # As above, but taking an array of functions and returning a single shlib function compile_shlib(funcs::Union{Array,Tuple}, path::String="./"; filename = "libfoo", demangle = true, cflags = ``, + target::StaticTarget=StaticTarget(), kwargs... ) for func in funcs @@ -185,17 +191,17 @@ function compile_shlib(funcs::Union{Array,Tuple}, path::String="./"; tt = Base.to_tuple_type(types) isconcretetype(tt) || error("input type signature `$types` is not concrete") - rt = last(only(native_code_typed(f, tt))) + rt = last(only(static_code_typed(f, tt; target, kwargs...))) isconcretetype(rt) || error("`$f$types` did not infer to a concrete type. Got `$rt`") nativetype = isprimitivetype(rt) || isa(rt, Ptr) nativetype || @warn "Return type `$rt` of `$f$types` does not appear to be a native type. Consider returning only a single value of a native machine type (i.e., a single float, int/uint, bool, or pointer). \n\nIgnoring this warning may result in Undefined Behavior!" end - generate_shlib(funcs, true, path, filename; demangle, cflags, kwargs...) + generate_shlib(funcs, path, filename; demangle, cflags, target, kwargs...) joinpath(abspath(path), filename * "." * Libdl.dlext) end - + """ ```julia @@ -281,12 +287,11 @@ generate_executable(f, tt, args...; kwargs...) = generate_executable(((f, tt),), function generate_executable(funcs::Union{Array,Tuple}, path=tempname(), name=fix_name(first(first(funcs))), filename=name; demangle = true, cflags = ``, + target::StaticTarget=StaticTarget(), kwargs... ) - lib_path = joinpath(path, "$filename.$(Libdl.dlext)") exec_path = joinpath(path, filename) - external = true - _, obj_path = generate_obj(funcs, external, path, filename; demangle, kwargs...) + _, obj_path = generate_obj(funcs, path, filename; demangle, target, kwargs...) # Pick a compiler cc = Sys.isapple() ? `cc` : clang() # Compile! @@ -318,8 +323,8 @@ end """ ```julia -generate_shlib(f::Function, tt, [external::Bool=true], [path::String], [name], [filename]; kwargs...) -generate_shlib(funcs::Array, [external::Bool=true], [path::String], [filename::String]; demangle=true, kwargs...) +generate_shlib(f::Function, tt, [path::String], [name], [filename]; kwargs...) +generate_shlib(funcs::Array, [path::String], [filename::String]; demangle=true, target::StaticTarget=StaticTarget(), kwargs...) ``` Low level interface for compiling a shared object / dynamically loaded library (`.so` / `.dylib`) for function `f` given a tuple type `tt` characterizing @@ -356,19 +361,20 @@ julia> ccall(("test", "example/test.dylib"), Float64, (Int64,), 100_000) 5.2564961094956075 ``` """ -function generate_shlib(f::Function, tt, external::Bool=true, path::String=tempname(), name=fix_name(f), filename=name; kwargs...) - generate_shlib(((f, tt),), external, path, filename; kwargs...) +function generate_shlib(f::Function, tt, path::String=tempname(), name=fix_name(f), filename=name; target=StaticTarget(), kwargs...) + generate_shlib(((f, tt),), path, filename; target, kwargs...) end # As above, but taking an array of functions and returning a single shlib -function generate_shlib(funcs::Union{Array,Tuple}, external::Bool=true, path::String=tempname(), filename::String="libfoo"; +function generate_shlib(funcs::Union{Array,Tuple}, path::String=tempname(), filename::String="libfoo"; demangle = true, cflags = ``, + target::StaticTarget=StaticTarget(), kwargs... ) lib_path = joinpath(path, "$filename.$(Libdl.dlext)") - _, obj_path = generate_obj(funcs, external, path, filename; demangle, kwargs...) + _, obj_path = generate_obj(funcs, path, filename; target, demangle, kwargs...) # Pick a Clang cc = Sys.isapple() ? `cc` : clang() # Compile! @@ -377,27 +383,27 @@ function generate_shlib(funcs::Union{Array,Tuple}, external::Bool=true, path::St path, name end -function native_code_llvm(@nospecialize(func), @nospecialize(types); kwargs...) - job, kwargs = native_job(func, types, true; kwargs...) +function static_code_llvm(@nospecialize(func), @nospecialize(types); target::StaticTarget=StaticTarget(), kwargs...) + job, kwargs = static_job(func, types; target, kwargs...) GPUCompiler.code_llvm(stdout, job; kwargs...) end -function native_code_typed(@nospecialize(func), @nospecialize(types); kwargs...) - job, kwargs = native_job(func, types, true; kwargs...) +function static_code_typed(@nospecialize(func), @nospecialize(types); target::StaticTarget=StaticTarget(), kwargs...) + job, kwargs = static_job(func, types; target, kwargs...) GPUCompiler.code_typed(job; kwargs...) end -function native_code_native(@nospecialize(f), @nospecialize(tt), fname=fix_name(f); kwargs...) - job, kwargs = native_job(f, tt, true; fname, kwargs...) +function static_code_native(@nospecialize(f), @nospecialize(tt), fname=fix_name(f); target::StaticTarget=StaticTarget(), kwargs...) + job, kwargs = static_job(f, tt; fname, target, kwargs...) GPUCompiler.code_native(stdout, job; kwargs...) end # Return an LLVM module -function native_llvm_module(f, tt, name=fix_name(f); demangle, kwargs...) +function static_llvm_module(f, tt, name=fix_name(f); demangle, target::StaticTarget=StaticTarget(), kwargs...) if !demangle name = "julia_"*name end - job, kwargs = native_job(f, tt, true; name, kwargs...) + job, kwargs = static_job(f, tt; name, target, kwargs...) m = GPUCompiler.JuliaContext() do context m, _ = GPUCompiler.codegen(:llvm, job; strip=true, only_entry=false, validate=false) locate_pointers_and_runtime_calls(m) @@ -407,14 +413,14 @@ function native_llvm_module(f, tt, name=fix_name(f); demangle, kwargs...) end #Return an LLVM module for multiple functions -function native_llvm_module(funcs::Union{Array,Tuple}; demangle=true, kwargs...) +function static_llvm_module(funcs::Union{Array,Tuple}; demangle=true, target::StaticTarget=StaticTarget(), kwargs...) f,tt = funcs[1] mod = GPUCompiler.JuliaContext() do context name_f = fix_name(f) if !demangle name_f = "julia_"*name_f end - job, kwargs = native_job(f, tt, true; name = name_f, kwargs...) + job, kwargs = static_job(f, tt; name = name_f, target, kwargs...) mod,_ = GPUCompiler.codegen(:llvm, job; strip=true, only_entry=false, validate=false) if length(funcs) > 1 for func in funcs[2:end] @@ -423,7 +429,7 @@ function native_llvm_module(funcs::Union{Array,Tuple}; demangle=true, kwargs...) if !demangle name_f = "julia_"*name_f end - job, kwargs = native_job(f, tt, true; name = name_f, kwargs...) + job, kwargs = static_job(f, tt; name = name_f, target, kwargs...) tmod,_ = GPUCompiler.codegen(:llvm, job; strip=true, only_entry=false, validate=false) link!(mod,tmod) end @@ -458,7 +464,7 @@ end """ ```julia -generate_obj(f, tt, external::Bool, path::String = tempname(), filenamebase::String="obj"; +generate_obj(f, tt, path::String = tempname(), filenamebase::String="obj"; target = (), demangle = true, strip_llvm = false, @@ -498,7 +504,7 @@ end """ ```julia -generate_obj(funcs::Union{Array,Tuple}, external::Bool, path::String = tempname(), filenamebase::String="obj"; +generate_obj(funcs::Union{Array,Tuple}, path::String = tempname(), filenamebase::String="obj"; target = (), demangle =false, strip_llvm = false, @@ -514,18 +520,19 @@ which will be compiled. This is a named tuple with fields `triple`, `cpu`, and `features` (each of these are strings). The defaults compile to the native target. """ -function generate_obj(funcs::Union{Array,Tuple}, external::Bool, path::String = tempname(), filenamebase::String="obj"; +function generate_obj(funcs::Union{Array,Tuple}, path::String = tempname(), filenamebase::String="obj"; demangle = true, strip_llvm = false, strip_asm = true, opt_level = 3, + target::StaticTarget=StaticTarget(), kwargs...) f, tt = funcs[1] mkpath(path) obj_path = joinpath(path, "$filenamebase.o") - mod = native_llvm_module(funcs; demangle, kwargs...) + mod = static_llvm_module(funcs; demangle, kwargs...) obj = GPUCompiler.JuliaContext() do ctx - fakejob, _ = native_job(f, tt, external; kwargs...) + fakejob, _ = static_job(f, tt; target, kwargs...) obj, _ = GPUCompiler.emit_asm(fakejob, mod; strip=strip_asm, validate=false, format=LLVM.API.LLVMObjectFile) obj end diff --git a/src/target.jl b/src/target.jl index 777960a3..2f14e2b1 100644 --- a/src/target.jl +++ b/src/target.jl @@ -4,6 +4,31 @@ else const method_table = nothing end +""" +```julia + StaticTarget() # Native target + StaticTarget(platform::Base.BinaryPlatforms.Platform) # Specific target with generic CPU + StaticTarget(platform::Platform, cpu::String) # Specific target with specific CPU + StaticTarget(platform::Platform, cpu::String, features::String) # Specific target with specific CPU and features +``` +Struct that defines a target for the compilation +Beware that currently the compilation assumes that the code is on the host so platform specific code like: +```julia + Sys.isapple() ... +``` +does not behave as expected. +By default `StaticTarget()` is the native target. +""" +struct StaticTarget + platform::Platform + tm::LLVM.TargetMachine +end + +clean_triple(platform::Platform) = arch(platform) * "-" * os(platform) * libc_str(platform) +StaticTarget() = StaticTarget(HostPlatform(), unsafe_string(LLVM.API.LLVMGetHostCPUName()), unsafe_string(LLVM.API.LLVMGetHostCPUFeatures())) +StaticTarget(platform::Platform) = StaticTarget(platform, LLVM.TargetMachine(LLVM.Target(triple = clean_triple(platform)), clean_triple(platform))) +StaticTarget(platform::Platform, cpu::String) = StaticTarget(platform, LLVM.TargetMachine(LLVM.Target(triple = clean_triple(platform)), clean_triple(platform), cpu)) +StaticTarget(platform::Platform, cpu::String, features::String) = StaticTarget(platform, LLVM.TargetMachine(LLVM.Target(triple = clean_triple(platform)), clean_triple(platform), cpu, features)) """ ```julia @@ -29,18 +54,12 @@ macro device_override(ex) return esc(code) end -Base.@kwdef struct NativeCompilerTarget{MT} <: GPUCompiler.AbstractCompilerTarget - triple::String=Sys.MACHINE - cpu::String=(LLVM.version() < v"8") ? "" : unsafe_string(LLVM.API.LLVMGetHostCPUName()) - features::String=(LLVM.version() < v"8") ? "" : unsafe_string(LLVM.API.LLVMGetHostCPUFeatures()) - method_table::MT = method_table -end - -Base.@kwdef struct ExternalNativeCompilerTarget{MT} <: GPUCompiler.AbstractCompilerTarget - triple::String=Sys.MACHINE - cpu::String=(LLVM.version() < v"8") ? "" : unsafe_string(LLVM.API.LLVMGetHostCPUName()) - features::String=(LLVM.version() < v"8") ? "" : unsafe_string(LLVM.API.LLVMGetHostCPUFeatures()) - method_table::MT = method_table +# Default to native +struct StaticCompilerTarget{MT} <: GPUCompiler.AbstractCompilerTarget + triple::String + cpu::String + features::String + method_table::MT end module StaticRuntime @@ -53,58 +72,61 @@ module StaticRuntime report_exception_frame(idx, func, file, line) = return end -for target in (:NativeCompilerTarget, :ExternalNativeCompilerTarget) - @eval begin - GPUCompiler.llvm_triple(target::$target) = target.triple - function GPUCompiler.llvm_machine(target::$target) - triple = GPUCompiler.llvm_triple(target) +GPUCompiler.llvm_triple(target::StaticCompilerTarget) = target.triple - t = LLVM.Target(triple=triple) +function GPUCompiler.llvm_machine(target::StaticCompilerTarget) + triple = GPUCompiler.llvm_triple(target) - tm = LLVM.TargetMachine(t, triple, target.cpu, target.features, reloc=LLVM.API.LLVMRelocPIC) - GPUCompiler.asm_verbosity!(tm, true) + t = LLVM.Target(triple=triple) - return tm - end + tm = LLVM.TargetMachine(t, triple, target.cpu, target.features, reloc=LLVM.API.LLVMRelocPIC) + GPUCompiler.asm_verbosity!(tm, true) - GPUCompiler.runtime_slug(job::GPUCompiler.CompilerJob{<:$target}) = "native_$(job.config.target.cpu)-$(hash(job.config.target.features))" + return tm +end - GPUCompiler.runtime_module(::GPUCompiler.CompilerJob{<:$target}) = StaticRuntime - GPUCompiler.runtime_module(::GPUCompiler.CompilerJob{<:$target, StaticCompilerParams}) = StaticRuntime +GPUCompiler.runtime_slug(job::GPUCompiler.CompilerJob{<:StaticCompilerTarget}) = "static_$(job.config.target.cpu)-$(hash(job.config.target.features))" +GPUCompiler.runtime_module(::GPUCompiler.CompilerJob{<:StaticCompilerTarget}) = StaticRuntime +GPUCompiler.runtime_module(::GPUCompiler.CompilerJob{<:StaticCompilerTarget, StaticCompilerParams}) = StaticRuntime - GPUCompiler.can_throw(job::GPUCompiler.CompilerJob{<:$target, StaticCompilerParams}) = true - GPUCompiler.can_throw(job::GPUCompiler.CompilerJob{<:$target}) = true - GPUCompiler.get_interpreter(job::GPUCompiler.CompilerJob{<:$target, StaticCompilerParams}) = - StaticInterpreter(job.config.params.cache, GPUCompiler.method_table(job), job.world, - GPUCompiler.inference_params(job), GPUCompiler.optimization_params(job)) - GPUCompiler.ci_cache(job::GPUCompiler.CompilerJob{<:$target, StaticCompilerParams}) = job.config.params.cache - GPUCompiler.method_table(@nospecialize(job::GPUCompiler.CompilerJob{<:$target})) = job.config.target.method_table - end -end +GPUCompiler.can_throw(job::GPUCompiler.CompilerJob{<:StaticCompilerTarget, StaticCompilerParams}) = true +GPUCompiler.can_throw(job::GPUCompiler.CompilerJob{<:StaticCompilerTarget}) = true + +GPUCompiler.get_interpreter(job::GPUCompiler.CompilerJob{<:StaticCompilerTarget, StaticCompilerParams}) = + StaticInterpreter(job.config.params.cache, GPUCompiler.method_table(job), job.world, + GPUCompiler.inference_params(job), GPUCompiler.optimization_params(job)) +GPUCompiler.ci_cache(job::GPUCompiler.CompilerJob{<:StaticCompilerTarget, StaticCompilerParams}) = job.config.params.cache +GPUCompiler.method_table(@nospecialize(job::GPUCompiler.CompilerJob{<:StaticCompilerTarget})) = job.config.target.method_table -function native_job(@nospecialize(func::Function), @nospecialize(types::Type), external::Bool; + +function static_job(@nospecialize(func::Function), @nospecialize(types::Type); name = fix_name(func), kernel::Bool = false, - target = (;), + target::StaticTarget = StaticTarget(), method_table=method_table, kwargs... ) - target = merge(target, (;method_table)) source = methodinstance(typeof(func), Base.to_tuple_type(types)) - target = external ? ExternalNativeCompilerTarget(;target...) : NativeCompilerTarget(;target...) + tm = target.tm + gputarget = StaticCompilerTarget(LLVM.triple(tm), LLVM.cpu(tm), LLVM.features(tm), method_table) params = StaticCompilerParams() - config = GPUCompiler.CompilerConfig(target, params, name = name, kernel = kernel) + config = GPUCompiler.CompilerConfig(gputarget, params, name = name, kernel = kernel) StaticCompiler.CompilerJob(source, config), kwargs end - -function native_job(@nospecialize(func), @nospecialize(types), external; kernel::Bool=false, name=fix_name(repr(func)), target = (;), method_table=method_table, kwargs...) - target = merge(target, (; method_table)) +function static_job(@nospecialize(func), @nospecialize(types); + name = fix_name(func), + kernel::Bool = false, + target::StaticTarget = StaticTarget(), + method_table=method_table, + kwargs... +) source = methodinstance(typeof(func), Base.to_tuple_type(types)) - target = external ? ExternalNativeCompilerTarget(;target...) : NativeCompilerTarget(;target...) + tm = target.tm + gputarget = StaticCompilerTarget(LLVM.triple(tm), LLVM.cpu(tm), LLVM.features(tm), method_table) params = StaticCompilerParams() - config = GPUCompiler.CompilerConfig(target, params, name = name, kernel = kernel) - GPUCompiler.CompilerJob(source, config), kwargs -end + config = GPUCompiler.CompilerConfig(gputarget, params, name = name, kernel = kernel) + StaticCompiler.CompilerJob(source, config), kwargs +end \ No newline at end of file diff --git a/test/scripts/interop.jl b/test/scripts/interop.jl index b601e090..24546ec1 100644 --- a/test/scripts/interop.jl +++ b/test/scripts/interop.jl @@ -13,4 +13,4 @@ function interop(argc, argv) end # Attempt to compile -path = compile_executable(interop, (Int64, Ptr{Ptr{UInt8}}), "./", cflags=`-ldl -lm`) +path = compile_executable(interop, (Int64, Ptr{Ptr{UInt8}}), "./", c_flags=`-ldl -lm`) From 0e0f8722ab468e41392aa188fa0fa5bebb81d409 Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Wed, 29 Nov 2023 11:13:13 -0300 Subject: [PATCH 2/7] Fixup macos issues --- src/StaticCompiler.jl | 4 ++-- src/target.jl | 2 +- test/scripts/interop.jl | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/StaticCompiler.jl b/src/StaticCompiler.jl index 6bca325e..2d67382b 100644 --- a/src/StaticCompiler.jl +++ b/src/StaticCompiler.jl @@ -12,7 +12,7 @@ using LLD_jll: lld using StaticTools using StaticTools: @symbolcall, @c_str, println using Core: MethodTable -using Base:BinaryPlatforms.Platform, BinaryPlatforms.HostPlatform, BinaryPlatforms.arch, BinaryPlatforms.os, BinaryPlatforms.libc_str +using Base:BinaryPlatforms.Platform, BinaryPlatforms.HostPlatform, BinaryPlatforms.arch, BinaryPlatforms.os_str, BinaryPlatforms.libc_str export load_function, compile_shlib, compile_executable export static_code_llvm, static_code_typed, static_llvm_module, static_code_native @@ -399,7 +399,7 @@ function static_code_native(@nospecialize(f), @nospecialize(tt), fname=fix_name( end # Return an LLVM module -function static_llvm_module(f, tt, name=fix_name(f); demangle, target::StaticTarget=StaticTarget(), kwargs...) +function static_llvm_module(f, tt, name=fix_name(f); demangle=true, target::StaticTarget=StaticTarget(), kwargs...) if !demangle name = "julia_"*name end diff --git a/src/target.jl b/src/target.jl index 2f14e2b1..26d5c697 100644 --- a/src/target.jl +++ b/src/target.jl @@ -24,7 +24,7 @@ struct StaticTarget tm::LLVM.TargetMachine end -clean_triple(platform::Platform) = arch(platform) * "-" * os(platform) * libc_str(platform) +clean_triple(platform::Platform) = arch(platform) * os_str(platform) * libc_str(platform) StaticTarget() = StaticTarget(HostPlatform(), unsafe_string(LLVM.API.LLVMGetHostCPUName()), unsafe_string(LLVM.API.LLVMGetHostCPUFeatures())) StaticTarget(platform::Platform) = StaticTarget(platform, LLVM.TargetMachine(LLVM.Target(triple = clean_triple(platform)), clean_triple(platform))) StaticTarget(platform::Platform, cpu::String) = StaticTarget(platform, LLVM.TargetMachine(LLVM.Target(triple = clean_triple(platform)), clean_triple(platform), cpu)) diff --git a/test/scripts/interop.jl b/test/scripts/interop.jl index 24546ec1..422f682a 100644 --- a/test/scripts/interop.jl +++ b/test/scripts/interop.jl @@ -13,4 +13,4 @@ function interop(argc, argv) end # Attempt to compile -path = compile_executable(interop, (Int64, Ptr{Ptr{UInt8}}), "./", c_flags=`-ldl -lm`) +path = compile_executable(interop, (Int64, Ptr{Ptr{UInt8}}), "./",cflags=`-ldl -lm`) From 0d537cdadafa9236d7838ea9854ccd9e68a92f53 Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Wed, 29 Nov 2023 12:09:47 -0300 Subject: [PATCH 3/7] Add wasm test and fixup macos --- src/StaticCompiler.jl | 11 ++++++----- src/target.jl | 3 ++- test/scripts/wasm.jl | 16 ++++++++++++++++ 3 files changed, 24 insertions(+), 6 deletions(-) create mode 100644 test/scripts/wasm.jl diff --git a/src/StaticCompiler.jl b/src/StaticCompiler.jl index 2d67382b..b84cc2f8 100644 --- a/src/StaticCompiler.jl +++ b/src/StaticCompiler.jl @@ -17,6 +17,7 @@ using Base:BinaryPlatforms.Platform, BinaryPlatforms.HostPlatform, BinaryPlatfor export load_function, compile_shlib, compile_executable export static_code_llvm, static_code_typed, static_llvm_module, static_code_native export @device_override, @print_and_throw +export StaticTarget include("interpreter.jl") include("target.jl") @@ -465,7 +466,7 @@ end """ ```julia generate_obj(f, tt, path::String = tempname(), filenamebase::String="obj"; - target = (), + target::StaticTarget=StaticTarget(), demangle = true, strip_llvm = false, strip_asm = true, @@ -477,7 +478,7 @@ a tuple type `tt` characterizing the types of the arguments for which the function will be compiled. `target` can be used to change the output target. This is useful for compiling to WebAssembly and embedded targets. -This is a named tuple with fields `triple`, `cpu`, and `features` (each of these are strings). +This is a struct of the type StaticTarget() The defaults compile to the native target. If `demangle` is set to `false`, compiled function names are prepended with "julia_". @@ -487,7 +488,7 @@ If `demangle` is set to `false`, compiled function names are prepended with "jul julia> fib(n) = n <= 1 ? n : fib(n - 1) + fib(n - 2) fib (generic function with 1 method) -julia> path, name, table = StaticCompiler.generate_obj_for_compile(fib, Tuple{Int64}, "./test") +julia> path, name, table = StaticCompiler.generate_obj(fib, Tuple{Int64}, "./test") ("./test", "fib", IdDict{Any, String}()) shell> tree \$path @@ -505,7 +506,7 @@ end """ ```julia generate_obj(funcs::Union{Array,Tuple}, path::String = tempname(), filenamebase::String="obj"; - target = (), + target::StaticTarget=StaticTarget(), demangle =false, strip_llvm = false, strip_asm = true, @@ -517,7 +518,7 @@ Low level interface for compiling object code (`.o`) for an array of Tuples which will be compiled. `target` can be used to change the output target. This is useful for compiling to WebAssembly and embedded targets. -This is a named tuple with fields `triple`, `cpu`, and `features` (each of these are strings). +This is a struct of the type StaticTarget() The defaults compile to the native target. """ function generate_obj(funcs::Union{Array,Tuple}, path::String = tempname(), filenamebase::String="obj"; diff --git a/src/target.jl b/src/target.jl index 26d5c697..3d545637 100644 --- a/src/target.jl +++ b/src/target.jl @@ -20,7 +20,7 @@ does not behave as expected. By default `StaticTarget()` is the native target. """ struct StaticTarget - platform::Platform + platform::Union{Platform,Nothing} tm::LLVM.TargetMachine end @@ -29,6 +29,7 @@ StaticTarget() = StaticTarget(HostPlatform(), unsafe_string(LLVM.API.LLVMGetHost StaticTarget(platform::Platform) = StaticTarget(platform, LLVM.TargetMachine(LLVM.Target(triple = clean_triple(platform)), clean_triple(platform))) StaticTarget(platform::Platform, cpu::String) = StaticTarget(platform, LLVM.TargetMachine(LLVM.Target(triple = clean_triple(platform)), clean_triple(platform), cpu)) StaticTarget(platform::Platform, cpu::String, features::String) = StaticTarget(platform, LLVM.TargetMachine(LLVM.Target(triple = clean_triple(platform)), clean_triple(platform), cpu, features)) +StaticTarget(triple::String, cpu::String, features::String) = StaticTarget(nothing, LLVM.TargetMachine(LLVM.Target(triple = triple), triple, cpu, features)) """ ```julia diff --git a/test/scripts/wasm.jl b/test/scripts/wasm.jl new file mode 100644 index 00000000..9222660a --- /dev/null +++ b/test/scripts/wasm.jl @@ -0,0 +1,16 @@ +# Test that we can compile an object to wasm +# WebAssemblyCompiler.jl is a better tool for this, but this exercises the cross compilation pipeline + +using StaticCompiler +using LLVM +InitializeAllTargets() +InitializeAllTargetInfos() +InitializeAllAsmPrinters() +InitializeAllAsmParsers() +InitializeAllTargetMCs() + +fib(n) = n <= 1 ? n : fib(n - 1) + fib(n - 2) + +target=StaticTarget("wasm32","","") + +StaticCompiler.generate_obj(fib, Tuple{Int64}, "./test", target=target) \ No newline at end of file From bf0740bf858d2ad025e9a06bbcdad6a14c60992d Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Wed, 29 Nov 2023 12:11:05 -0300 Subject: [PATCH 4/7] Whitespace --- test/scripts/interop.jl | 2 +- test/scripts/wasm.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/scripts/interop.jl b/test/scripts/interop.jl index 422f682a..b601e090 100644 --- a/test/scripts/interop.jl +++ b/test/scripts/interop.jl @@ -13,4 +13,4 @@ function interop(argc, argv) end # Attempt to compile -path = compile_executable(interop, (Int64, Ptr{Ptr{UInt8}}), "./",cflags=`-ldl -lm`) +path = compile_executable(interop, (Int64, Ptr{Ptr{UInt8}}), "./", cflags=`-ldl -lm`) diff --git a/test/scripts/wasm.jl b/test/scripts/wasm.jl index 9222660a..beff3b7d 100644 --- a/test/scripts/wasm.jl +++ b/test/scripts/wasm.jl @@ -13,4 +13,4 @@ fib(n) = n <= 1 ? n : fib(n - 1) + fib(n - 2) target=StaticTarget("wasm32","","") -StaticCompiler.generate_obj(fib, Tuple{Int64}, "./test", target=target) \ No newline at end of file +StaticCompiler.generate_obj(fib, Tuple{Int64}, "./test", target=target) From 2bf702662d1bce18d925a0d4130602dfd7b52de1 Mon Sep 17 00:00:00 2001 From: gbaraldi Date: Tue, 5 Dec 2023 13:15:18 -0500 Subject: [PATCH 5/7] Add way to set compiler for cross target and also bump major version --- Project.toml | 2 +- src/StaticCompiler.jl | 22 +++++++++++++++++----- src/target.jl | 29 ++++++++++++++++++++++++----- 3 files changed, 42 insertions(+), 11 deletions(-) diff --git a/Project.toml b/Project.toml index 4be53224..8daa749f 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "StaticCompiler" uuid = "81625895-6c0f-48fc-b932-11a18313743c" authors = ["Tom Short and contributors"] -version = "0.6.2" +version = "0.7.0" [deps] Clang_jll = "0ee61d77-7f21-5576-8119-9fcc46b10100" diff --git a/src/StaticCompiler.jl b/src/StaticCompiler.jl index b84cc2f8..e0fc69c6 100644 --- a/src/StaticCompiler.jl +++ b/src/StaticCompiler.jl @@ -13,7 +13,7 @@ using StaticTools using StaticTools: @symbolcall, @c_str, println using Core: MethodTable using Base:BinaryPlatforms.Platform, BinaryPlatforms.HostPlatform, BinaryPlatforms.arch, BinaryPlatforms.os_str, BinaryPlatforms.libc_str - +using Base:BinaryPlatforms.platform_dlext export load_function, compile_shlib, compile_executable export static_code_llvm, static_code_typed, static_llvm_module, static_code_native export @device_override, @print_and_throw @@ -294,7 +294,12 @@ function generate_executable(funcs::Union{Array,Tuple}, path=tempname(), name=fi exec_path = joinpath(path, filename) _, obj_path = generate_obj(funcs, path, filename; demangle, target, kwargs...) # Pick a compiler - cc = Sys.isapple() ? `cc` : clang() + if !isnothing(target.compiler) + cc = `$(target.compiler)` + else + cc = Sys.isapple() ? `cc` : clang() + end + # Compile! if Sys.isapple() # Apple no longer uses _start, so we can just specify a custom entry @@ -372,12 +377,19 @@ function generate_shlib(funcs::Union{Array,Tuple}, path::String=tempname(), file target::StaticTarget=StaticTarget(), kwargs... ) - - lib_path = joinpath(path, "$filename.$(Libdl.dlext)") + if !isnothing(target.platform) + lib_path = joinpath(path, "$filename.$(platform_dlext(target.platform))") + else + lib_path = joinpath(path, "$filename.$(Libdl.dlext)") + end _, obj_path = generate_obj(funcs, path, filename; target, demangle, kwargs...) # Pick a Clang - cc = Sys.isapple() ? `cc` : clang() + if !isnothing(target.compiler) + cc = `$(target.compiler)` + else + cc = Sys.isapple() ? `cc` : clang() + end # Compile! run(`$cc -shared $cflags $obj_path -o $lib_path `) diff --git a/src/target.jl b/src/target.jl index 3d545637..df53b9c2 100644 --- a/src/target.jl +++ b/src/target.jl @@ -18,18 +18,37 @@ Beware that currently the compilation assumes that the code is on the host so pl ``` does not behave as expected. By default `StaticTarget()` is the native target. + +For cross-compilation of executables and shared libraries, one also needs to call `set_compiler!` with the path to a valid C compiler +for the target platform. For example, to cross-compile for aarch64 using a compiler from homebrew, one can use: +```julia + set_compiler!(StaticTarget(parse(Platform,"aarch64-gnu-linux")), "/opt/homebrew/bin/aarch64-unknown-linux-gnu-gcc") +``` """ -struct StaticTarget +mutable struct StaticTarget platform::Union{Platform,Nothing} tm::LLVM.TargetMachine + compiler::Union{String,Nothing} end clean_triple(platform::Platform) = arch(platform) * os_str(platform) * libc_str(platform) StaticTarget() = StaticTarget(HostPlatform(), unsafe_string(LLVM.API.LLVMGetHostCPUName()), unsafe_string(LLVM.API.LLVMGetHostCPUFeatures())) -StaticTarget(platform::Platform) = StaticTarget(platform, LLVM.TargetMachine(LLVM.Target(triple = clean_triple(platform)), clean_triple(platform))) -StaticTarget(platform::Platform, cpu::String) = StaticTarget(platform, LLVM.TargetMachine(LLVM.Target(triple = clean_triple(platform)), clean_triple(platform), cpu)) -StaticTarget(platform::Platform, cpu::String, features::String) = StaticTarget(platform, LLVM.TargetMachine(LLVM.Target(triple = clean_triple(platform)), clean_triple(platform), cpu, features)) -StaticTarget(triple::String, cpu::String, features::String) = StaticTarget(nothing, LLVM.TargetMachine(LLVM.Target(triple = triple), triple, cpu, features)) +StaticTarget(platform::Platform) = StaticTarget(platform, LLVM.TargetMachine(LLVM.Target(triple = clean_triple(platform)), clean_triple(platform)), nothing) +StaticTarget(platform::Platform, cpu::String) = StaticTarget(platform, LLVM.TargetMachine(LLVM.Target(triple = clean_triple(platform)), clean_triple(platform), cpu), nothing) +StaticTarget(platform::Platform, cpu::String, features::String) = StaticTarget(platform, LLVM.TargetMachine(LLVM.Target(triple = clean_triple(platform)), clean_triple(platform), cpu, features), nothing) + +function StaticTarget(triple::String, cpu::String, features::String) + platform = tryparse(Platform, triple) + StaticTarget(platform, LLVM.TargetMachine(LLVM.Target(triple = triple), triple, cpu, features), nothing) +end + +""" +Set the compiler for cross compilation + ```julia + set_compiler!(StaticTarget(parse(Platform,"aarch64-gnu-linux")), "/opt/homebrew/bin/aarch64-elf-gcc") +``` +""" +set_compiler!(target::StaticTarget, compiler::String) = (target.compiler = compiler) """ ```julia From ac2d56f4d91962193b5ef809e27e428b1d225fef Mon Sep 17 00:00:00 2001 From: gbaraldi Date: Tue, 5 Dec 2023 13:25:39 -0500 Subject: [PATCH 6/7] Bump Bumper :) --- test/Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Project.toml b/test/Project.toml index 5498846c..a36e2086 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -16,4 +16,4 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" Bumper = "8ce10254-0962-460f-a3d8-1f77fea1446e" [compat] -Bumper = "0.5.1" \ No newline at end of file +Bumper = "0.6" \ No newline at end of file From e3412cb6be4766ef08c328e24f9b892085e4a804 Mon Sep 17 00:00:00 2001 From: gbaraldi Date: Tue, 5 Dec 2023 16:02:30 -0500 Subject: [PATCH 7/7] Prepare code for future endeavours into adding the GC --- src/StaticCompiler.jl | 10 +++++----- src/target.jl | 16 +++++++++++----- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/StaticCompiler.jl b/src/StaticCompiler.jl index e0fc69c6..7277b2f0 100644 --- a/src/StaticCompiler.jl +++ b/src/StaticCompiler.jl @@ -398,7 +398,7 @@ end function static_code_llvm(@nospecialize(func), @nospecialize(types); target::StaticTarget=StaticTarget(), kwargs...) job, kwargs = static_job(func, types; target, kwargs...) - GPUCompiler.code_llvm(stdout, job; kwargs...) + GPUCompiler.code_llvm(stdout, job; libraries=false, kwargs...) end function static_code_typed(@nospecialize(func), @nospecialize(types); target::StaticTarget=StaticTarget(), kwargs...) @@ -408,7 +408,7 @@ end function static_code_native(@nospecialize(f), @nospecialize(tt), fname=fix_name(f); target::StaticTarget=StaticTarget(), kwargs...) job, kwargs = static_job(f, tt; fname, target, kwargs...) - GPUCompiler.code_native(stdout, job; kwargs...) + GPUCompiler.code_native(stdout, job; libraries=false, kwargs...) end # Return an LLVM module @@ -418,7 +418,7 @@ function static_llvm_module(f, tt, name=fix_name(f); demangle=true, target::Stat end job, kwargs = static_job(f, tt; name, target, kwargs...) m = GPUCompiler.JuliaContext() do context - m, _ = GPUCompiler.codegen(:llvm, job; strip=true, only_entry=false, validate=false) + m, _ = GPUCompiler.codegen(:llvm, job; strip=true, only_entry=false, validate=false, libraries=false) locate_pointers_and_runtime_calls(m) m end @@ -434,7 +434,7 @@ function static_llvm_module(funcs::Union{Array,Tuple}; demangle=true, target::St name_f = "julia_"*name_f end job, kwargs = static_job(f, tt; name = name_f, target, kwargs...) - mod,_ = GPUCompiler.codegen(:llvm, job; strip=true, only_entry=false, validate=false) + mod,_ = GPUCompiler.codegen(:llvm, job; strip=true, only_entry=false, validate=false, libraries=false) if length(funcs) > 1 for func in funcs[2:end] f,tt = func @@ -443,7 +443,7 @@ function static_llvm_module(funcs::Union{Array,Tuple}; demangle=true, target::St name_f = "julia_"*name_f end job, kwargs = static_job(f, tt; name = name_f, target, kwargs...) - tmod,_ = GPUCompiler.codegen(:llvm, job; strip=true, only_entry=false, validate=false) + tmod,_ = GPUCompiler.codegen(:llvm, job; strip=true, only_entry=false, validate=false, libraries=false) link!(mod,tmod) end end diff --git a/src/target.jl b/src/target.jl index df53b9c2..5faec0d1 100644 --- a/src/target.jl +++ b/src/target.jl @@ -29,13 +29,14 @@ mutable struct StaticTarget platform::Union{Platform,Nothing} tm::LLVM.TargetMachine compiler::Union{String,Nothing} + julia_runtime::Bool end clean_triple(platform::Platform) = arch(platform) * os_str(platform) * libc_str(platform) StaticTarget() = StaticTarget(HostPlatform(), unsafe_string(LLVM.API.LLVMGetHostCPUName()), unsafe_string(LLVM.API.LLVMGetHostCPUFeatures())) -StaticTarget(platform::Platform) = StaticTarget(platform, LLVM.TargetMachine(LLVM.Target(triple = clean_triple(platform)), clean_triple(platform)), nothing) -StaticTarget(platform::Platform, cpu::String) = StaticTarget(platform, LLVM.TargetMachine(LLVM.Target(triple = clean_triple(platform)), clean_triple(platform), cpu), nothing) -StaticTarget(platform::Platform, cpu::String, features::String) = StaticTarget(platform, LLVM.TargetMachine(LLVM.Target(triple = clean_triple(platform)), clean_triple(platform), cpu, features), nothing) +StaticTarget(platform::Platform) = StaticTarget(platform, LLVM.TargetMachine(LLVM.Target(triple = clean_triple(platform)), clean_triple(platform)), nothing, false) +StaticTarget(platform::Platform, cpu::String) = StaticTarget(platform, LLVM.TargetMachine(LLVM.Target(triple = clean_triple(platform)), clean_triple(platform), cpu), nothing, false) +StaticTarget(platform::Platform, cpu::String, features::String) = StaticTarget(platform, LLVM.TargetMachine(LLVM.Target(triple = clean_triple(platform)), clean_triple(platform), cpu, features), nothing, false) function StaticTarget(triple::String, cpu::String, features::String) platform = tryparse(Platform, triple) @@ -50,6 +51,9 @@ Set the compiler for cross compilation """ set_compiler!(target::StaticTarget, compiler::String) = (target.compiler = compiler) + +set_runtime!(target::StaticTarget, julia_runtime::Bool) = (target.julia_runtime = julia_runtime) + """ ```julia @device_override old_bad_method(arg1::Type1, arg2::Type2) = new_good_method(arg1, arg2) @@ -79,6 +83,7 @@ struct StaticCompilerTarget{MT} <: GPUCompiler.AbstractCompilerTarget triple::String cpu::String features::String + julia_runtime::Bool method_table::MT end @@ -115,6 +120,7 @@ GPUCompiler.runtime_module(::GPUCompiler.CompilerJob{<:StaticCompilerTarget, Sta GPUCompiler.can_throw(job::GPUCompiler.CompilerJob{<:StaticCompilerTarget, StaticCompilerParams}) = true GPUCompiler.can_throw(job::GPUCompiler.CompilerJob{<:StaticCompilerTarget}) = true +GPUCompiler.uses_julia_runtime(job::GPUCompiler.CompilerJob{<:StaticCompilerTarget}) = job.config.target.julia_runtime GPUCompiler.get_interpreter(job::GPUCompiler.CompilerJob{<:StaticCompilerTarget, StaticCompilerParams}) = StaticInterpreter(job.config.params.cache, GPUCompiler.method_table(job), job.world, GPUCompiler.inference_params(job), GPUCompiler.optimization_params(job)) @@ -131,7 +137,7 @@ function static_job(@nospecialize(func::Function), @nospecialize(types::Type); ) source = methodinstance(typeof(func), Base.to_tuple_type(types)) tm = target.tm - gputarget = StaticCompilerTarget(LLVM.triple(tm), LLVM.cpu(tm), LLVM.features(tm), method_table) + gputarget = StaticCompilerTarget(LLVM.triple(tm), LLVM.cpu(tm), LLVM.features(tm), target.julia_runtime, method_table) params = StaticCompilerParams() config = GPUCompiler.CompilerConfig(gputarget, params, name = name, kernel = kernel) StaticCompiler.CompilerJob(source, config), kwargs @@ -145,7 +151,7 @@ function static_job(@nospecialize(func), @nospecialize(types); ) source = methodinstance(typeof(func), Base.to_tuple_type(types)) tm = target.tm - gputarget = StaticCompilerTarget(LLVM.triple(tm), LLVM.cpu(tm), LLVM.features(tm), method_table) + gputarget = StaticCompilerTarget(LLVM.triple(tm), LLVM.cpu(tm), LLVM.features(tm), target.julia_runtime, method_table) params = StaticCompilerParams() config = GPUCompiler.CompilerConfig(gputarget, params, name = name, kernel = kernel) StaticCompiler.CompilerJob(source, config), kwargs