Skip to content

Commit

Permalink
Merge pull request #21 from ocurrent/sandmark-dependencies
Browse files Browse the repository at this point in the history 
Adding ocaml-bench/sandmark dependency packages
  • Loading branch information
@moyodiallo
moyodiallo committed Jun 28, 2022
2 parents 65cb34c + 8e6593c commit ce5ec41
Show file tree
Hide file tree
Showing 14 changed files with 418 additions and 117 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,4 @@ var
.idea
.vscode
capnp-secrets/
tags
88 changes: 60 additions & 28 deletions lib/analyse.ml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,12 @@ let job_log job =

let make_placeholder_selections ~platforms ~opam_repository_commits =
platforms |> List.map (fun platform ->
{ Selection.variant = fst platform; packages = []; commit = Current_git.Commit_id.hash (List.hd opam_repository_commits); command=None }
{
Selection.variant = fst platform;
packages = [];
commits = List.map (fun y -> Current_git.Commit_id.repo y, Current_git.Commit_id.hash y) opam_repository_commits;
command=None
}
)

(* Remove all the *.dev packages in opam_files from the given selections *)
Expand All @@ -72,7 +77,6 @@ let remove_dev_selections ~opam_files = function
`Opam_build (remove_dev_selections_from_opam_build ~opam_files selections)
| `Opam_monorepo _ as x -> x
| `Not_opam _ as x -> x

module Analysis = struct
type t = {
opam_files : string list;
Expand Down Expand Up @@ -208,7 +212,7 @@ module Analysis = struct
let pkg = Str.global_replace dev_re "" pkg_name in
let opam = OpamFile.OPAM.read_from_string opam_str in
let command = build_command_to_string ~pkg (OpamFile.OPAM.build opam) in
{ Selection.variant=selection.Selection.variant; packages=selection.packages; commit=selection.commit; command=command }
{ Selection.variant=selection.Selection.variant; packages=selection.packages; commits=selection.commits; command=command }
| _ ->
selection

Expand Down Expand Up @@ -269,17 +273,7 @@ module Analysis = struct
| Ok x -> Ok (List.map Selection.of_worker x)
| Error (`Msg msg) -> Fmt.error_msg "Error from solver: %s" msg

let of_dir ~solver ~job ~platforms ~opam_repository_commits ~package_name ?is_compiler ?compiler_commit dir =
let is_compiler = Option.value is_compiler ~default:false in
Current.Job.log job
"Analysing %s: @[<v>platforms=@[%a@]@,opam_repository_commits=@[%a@]@,is_compiler=%a@,compiler_commit=%a@]"
package_name
pp_platforms platforms
(Fmt.list Git.Commit_id.pp) opam_repository_commits
Fmt.bool is_compiler
(Fmt.option Git.Commit_id.pp) compiler_commit;
let solve = solve ~opam_repository_commits ~job ~solver in
let ty = type_of_dir dir in
let find_opam_files ~job ~dir =
let cmd = "", [| "find"; "."; "-maxdepth"; "3"; "-name"; "*.opam" |] in
Current.Process.check_output ~cwd:dir ~cancellable:true ~job cmd >>!= fun output ->
let opam_files =
Expand Down Expand Up @@ -309,7 +303,20 @@ module Analysis = struct
Some path
else None
)
in
in Lwt_result.return opam_files

let of_dir ~solver ~job ~platforms ~opam_repository_commits ~package_name ?is_compiler ?compiler_commit dir =
let is_compiler = Option.value is_compiler ~default:false in
Current.Job.log job
"Analysing %s: @[<v>platforms=@[%a@]@,opam_repository_commits=@[%a@]@,is_compiler=%a@,compiler_commit=%a@]"
package_name
pp_platforms platforms
(Fmt.list Git.Commit_id.pp) opam_repository_commits
Fmt.bool is_compiler
(Fmt.option Git.Commit_id.pp) compiler_commit;
let solve = solve ~opam_repository_commits ~job ~solver in
let ty = type_of_dir dir in
find_opam_files ~job ~dir >>!= fun opam_files ->
Analyse_ocamlformat.get_ocamlformat_source job ~opam_files ~root:dir >>!= fun ocamlformat_source ->
if is_compiler || opam_files = [] then Lwt_result.return (
dummy_analysis ~platforms ~opam_repository_commits ~package_name
Expand All @@ -334,6 +341,17 @@ module Analysis = struct
Current.Job.log job "@[<v2>Results:@,%a@]" Yojson.Safe.(pretty_print ~std:true) (to_yojson r);
Lwt_result.return r
)

let of_dir_sandmark ~job ~platforms ~opam_repository_commits ~package_name ?is_compiler ?compiler_commit () =
let is_compiler = Option.value is_compiler ~default:false in
Current.Job.log job
"Analysing %s: @[<v>platforms=@[%a@]@,opam_repository_commits=@[%a@]@,is_compiler=%a@,compiler_commit=%a@]"
package_name
pp_platforms platforms
(Fmt.list Git.Commit_id.pp) opam_repository_commits
Fmt.bool is_compiler
(Fmt.option Git.Commit_id.pp) compiler_commit;
Lwt_result.return (dummy_analysis ~platforms ~opam_repository_commits ~package_name)
end

let platform_to_yojson (variant, vars) =
Expand All @@ -350,12 +368,17 @@ module Examine = struct

module Key = struct
type t = {
sandmark_package: string option;
src: Current_git.Commit.t;
compiler_commit : Current_git.Commit_id.t option;
}

let digest { src; compiler_commit } =
let digest { sandmark_package; src; compiler_commit } =
let json = `Assoc [
"sandmark_package",
(match sandmark_package with
| None -> `Null
| Some s -> `String s);
"src", commit_to_yojson src;
"compiler_commit",
match compiler_commit with
Expand All @@ -372,9 +395,10 @@ module Examine = struct
platforms : (Variant.t * Worker.Vars.t) list;
is_compiler : bool;
compiler_commit : Current_git.Commit_id.t option;
sandmark_package: string option
}

let digest { opam_repository_commits; platforms; is_compiler; compiler_commit } =
let digest { opam_repository_commits; platforms; is_compiler; compiler_commit; sandmark_package } =
let json = `Assoc [
"opam-repositories",
commit_ids_to_yojson opam_repository_commits;
Expand All @@ -383,9 +407,13 @@ module Examine = struct
"is_compiler",
`Bool is_compiler;
"compiler_commit",
match compiler_commit with
(match compiler_commit with
| None -> `Null
| Some cc -> commit_id_to_yojson cc
| Some cc -> commit_id_to_yojson cc);
"sandmark_package",
(match sandmark_package with
| None -> `Null
| Some s -> `String s)
]
in
Yojson.Safe.to_string json
Expand All @@ -395,11 +423,14 @@ module Examine = struct

let id = "ci-analyse"

let run solver job { Key.src; _ } { Value.opam_repository_commits; platforms; is_compiler; compiler_commit } =
let run solver job { Key.src; _ } { Value.opam_repository_commits; platforms; is_compiler; compiler_commit; sandmark_package } =
let package_name = package_name_from_commit src in
Current.Job.start job ~pool ~level:Current.Level.Harmless >>= fun () ->
Current_git.with_checkout ~job src @@ fun src ->
Analysis.of_dir ~solver ~platforms ~opam_repository_commits ~job ~package_name ~is_compiler ?compiler_commit src
match sandmark_package with
| None -> Analysis.of_dir ~solver ~platforms ~opam_repository_commits ~job ~package_name ~is_compiler ?compiler_commit src
| Some package_name ->
Analysis.of_dir_sandmark ~platforms ~opam_repository_commits ~job ~package_name ~is_compiler ?compiler_commit ()

let pp f (k, v) = Fmt.pf f "Analyse %s %s" (Key.digest k) (Value.digest v)

Expand Down Expand Up @@ -446,9 +477,9 @@ let split_label = function
| [a; b] -> (a, b)
| _ -> (s, "")

let examine ?label ~solver ~platforms ~opam_repository_commits ~is_compiler ~get_is_compiler_blocklisted ~repo src =
let examine ?label ?sandmark_package ~solver ~platforms ~opam_repository_commits ~is_compiler ~get_is_compiler_blocklisted ~repo src =
let (label1, label2) = split_label label in
Current.component "Analyse@ %s@ %s" label1 label2 |>
Current.component "Analyse@ %s@ %s" label1 label2|>
let> src = src
and> repo = repo
and> opam_repository_commits = opam_repository_commits
Expand All @@ -457,16 +488,17 @@ let examine ?label ~solver ~platforms ~opam_repository_commits ~is_compiler ~get
let platforms = platforms_for_package ~is_compiler ~get_is_compiler_blocklisted ~platforms package_name |> remap_platforms in
let opam_repository_commits = opam_repos_for_package ~is_compiler opam_repository_commits in
Examine_cache.run solver
{ src; compiler_commit=None }
{ Examine.Value.opam_repository_commits; platforms; is_compiler; compiler_commit=None }
{ sandmark_package; src; compiler_commit=None }
{ Examine.Value.opam_repository_commits; platforms; is_compiler; compiler_commit=None; sandmark_package}

let examine_with_compiler ?label ~solver ~platforms ~opam_repository_commits ~compiler_commit src =
let examine_with_compiler ?label ?sandmark_package ~solver ~platforms ~opam_repository_commits ~compiler_commit src =
Current.component "Analysis@ %a" Fmt.(option string) label |>
let> src = src
and> compiler_commit = compiler_commit
and> opam_repository_commits = opam_repository_commits
and> platforms = platforms in
let platforms = first_invariant_platform platforms |> remap_platforms in
Examine_cache.run solver
{ src; compiler_commit=(Some compiler_commit) }
{ Examine.Value.opam_repository_commits; platforms; is_compiler=false; compiler_commit=(Some compiler_commit) }
{ sandmark_package; src; compiler_commit=(Some compiler_commit) }
{ Examine.Value.opam_repository_commits; platforms; is_compiler=false; compiler_commit=(Some compiler_commit);
sandmark_package }
2 changes: 2 additions & 0 deletions lib/analyse.mli
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ end

val examine :
?label:string ->
?sandmark_package: string ->
solver:Ocaml_multicore_ci_api.Solver.t ->
platforms:Platform.t list Current.t ->
opam_repository_commits:Current_git.Commit_id.t list Current.t ->
Expand All @@ -37,6 +38,7 @@ val examine :

val examine_with_compiler :
?label:string ->
?sandmark_package:string ->
solver:Ocaml_multicore_ci_api.Solver.t ->
platforms:Platform.t list Current.t ->
opam_repository_commits:Current_git.Commit_id.t list Current.t ->
Expand Down
21 changes: 12 additions & 9 deletions lib/cluster_build.ml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,16 +28,18 @@ module Op = struct
repo : string; (* Used to choose a build cache *)
test_repo : string option; (* The repo under test, if repo is a compiler *)
label : string; (* A unique ID for this build within the commit *)
sandmark_package: string option (*A package name which we want to build or install, this is helpful if we want to use this build for a specific package instead of the repo*)
}

let to_json { pool; commit; compiler_commit; label; repo; test_repo } =
let to_json { pool; commit; compiler_commit; label; repo; test_repo; sandmark_package } =
`Assoc [
"pool", `String pool;
"commit", `String (Current_git.Commit_id.hash commit);
"compiler_commit", (match compiler_commit with None -> `Null | Some compiler_commit -> `String (Current_git.Commit_id.hash compiler_commit));
"repo", `String repo;
"test_repo", (match test_repo with None -> `Null | Some r -> `String r);
"label", `String label;
"sandmark_package",(match sandmark_package with None -> `Null | Some p -> `String p);
]

let digest t = Yojson.Safe.to_string (to_json t)
Expand Down Expand Up @@ -69,7 +71,7 @@ module Op = struct
Variant.pp variant
(Spec.digest_of_ty ~variant ty)

let run t job { Key.pool; commit; compiler_commit; label = _; repo; test_repo } spec =
let run t job { Key.pool; commit; compiler_commit; label = _; repo; test_repo; _ } spec =
let { Value.base; variant; ty } = spec in
let build_spec = Build.make_build_spec ~base ~compiler_commit ~repo ~test_repo ~variant ~ty
in
Expand Down Expand Up @@ -102,8 +104,9 @@ module Op = struct
Capability.with_ref build_job (Current_ocluster.Connection.run_job ~job) >>!= fun (_ : string) ->
Lwt_result.return ()

let pp f ({ Key.pool; repo; test_repo; commit; compiler_commit; label }, _) =
Fmt.pf f "test %s %a %a %a (%s:%s)"
let pp f ({ Key.pool; repo; test_repo; commit; compiler_commit; label; sandmark_package }, _) =
Fmt.pf f "test %a %s %a %a %a (%s:%s)"
Fmt.(option string) sandmark_package
repo (Fmt.option Fmt.string) test_repo
Current_git.Commit_id.pp commit
(Fmt.option Current_git.Commit_id.pp) compiler_commit
Expand All @@ -119,16 +122,16 @@ let config ?timeout sr =
let connection = Current_ocluster.Connection.create sr in
{ connection; timeout }

let build t ~platforms ~spec ~repo ?test_repo ?compiler_commit commit =
Current.component "cluster build" |>
let build t ~platforms ~spec ~repo ?test_repo ?compiler_commit ?sandmark_package commit =
Current.component "cluster build@. %a" Fmt.(option string) sandmark_package |>
let> { Spec.variant; ty; label } = spec
and> commit = commit
and> compiler_commit = Current.option_seq compiler_commit
and> platforms = platforms
and> repo = repo in
match List.find_opt (fun p -> Variant.equal p.Platform.variant variant) platforms with
| Some { Platform.builder = _; pool; variant; base; _ } ->
BC.run t { Op.Key.pool; commit; compiler_commit; repo; test_repo; label } { Op.Value.base; ty; variant }
BC.run t { Op.Key.pool; commit; compiler_commit; repo; test_repo; label; sandmark_package } { Op.Value.base; ty; variant }
| None ->
(* We can only get here if there is a bug. If the set of platforms changes, [Analyse] should recalculate. *)
let msg = Fmt.str "BUG: variant %a is not a supported platform" Variant.pp variant in
Expand All @@ -140,8 +143,8 @@ let get_job_id x =
| Some { Current.Metadata.job_id; _ } -> job_id
| None -> None

let v t ~platforms ~repo ?test_repo ?compiler_commit ~spec source =
let build = build t ~platforms ~spec ~repo ?test_repo ?compiler_commit source in
let v t ~platforms ~repo ?test_repo ?compiler_commit ?sandmark_package ~spec source =
let build = build t ~platforms ~spec ~repo ?test_repo ?compiler_commit ?sandmark_package source in
let+ state = Current.state ~hidden:true build
and+ job_id = get_job_id build
and+ spec = spec in
Expand Down
1 change: 1 addition & 0 deletions lib/cluster_build.mli
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ val v :
repo:string Current.t ->
?test_repo:string ->
?compiler_commit:Current_git.Commit_id.t Current.t ->
?sandmark_package:string ->
spec:Spec.t Current.t ->
Current_git.Commit_id.t Current.t ->
([> `Built | `Checked ] Current_term.Output.t * Current.job_id option) Current.t
Expand Down
16 changes: 12 additions & 4 deletions lib/opam_build.ml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,15 +113,23 @@ let install_os_deps selection =
comment "Preamble" :: result

let update_opam_repository selection =
let { Selection.commit; _ } = selection in
let { Selection.commits; _ } = selection in
let commits_in_order = List.rev commits in
let default = "https://github.com/ocaml/opam-repository.git" in
let commit,_ = List.partition (fun y -> String.equal default (fst y)) commits in
let _,commit = List.hd commit in
[
comment "Update opam-repository";
workdir "/home/opam/opam-repository";
run
"(git cat-file -e %s || git fetch origin master) && \
git reset -q --hard %s && git log --no-decorate -n1 --oneline \
&& opam update -u" commit commit;
]
git reset -q --hard %s && git log --no-decorate -n1 --oneline " commit commit;
] @
List.map (fun (repo,commit) ->
if String.equal repo default
then run "opam repo priority default 1 --set-default"
else run "opam repo add rep-%s %s --set-default" (String.sub commit 0 7) repo) commits_in_order @
[run "opam update -u"]

let copy_src =
[
Expand Down
Loading

0 comments on commit ce5ec41

Please sign in to comment.