diff --git a/.github/workflows/nix-toil-run-cpp.yml b/.github/workflows/nix-toil-run-cpp.yml new file mode 100644 index 0000000000..57613ab247 --- /dev/null +++ b/.github/workflows/nix-toil-run-cpp.yml @@ -0,0 +1,21 @@ +# (based on: https://github.com/cachix/install-nix-action readme) +name: "(Nix) toil run-cpp" +on: + pull_request: + push: +jobs: + toil-run-cpp: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + # Nix Flakes doesn't work on shallow clones + fetch-depth: 0 + - uses: cachix/install-nix-action@v12 + with: + nix_path: nixpkgs=channel:nixos-20.09 + # Enable Nix Flakes + install_url: https://github.com/numtide/nix-flakes-installer/releases/download/nix-2.4pre20201221_9fab14a/install + extra_nix_config: | + experimental-features = nix-command flakes + - run: nix build -L .#toil-run-cpp diff --git a/cpp/unit_tests.cc b/cpp/unit_tests.cc index 9971f6007f..0dfd946c1e 100644 --- a/cpp/unit_tests.cc +++ b/cpp/unit_tests.cc @@ -174,11 +174,11 @@ TEST libc_test() { ASSERT_EQ_FMT(10, result->at1(), "%d"); // This depends on the file system - auto files = libc::glob(new Str("*.py")); + auto files = libc::glob(new Str("*.mk")); ASSERT_EQ_FMT(1, len(files), "%d"); - print(files->index(0)); // should get __init__.py only + print(files->index(0)); // should get portable_rules.mk only - auto files2 = libc::glob(new Str("*.pyzzz")); + auto files2 = libc::glob(new Str("*.mkzzz")); ASSERT_EQ_FMT(0, len(files2), "%d"); Str* h = libc::gethostname(); diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000000..0a123bd776 --- /dev/null +++ b/flake.lock @@ -0,0 +1,61 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1609444776, + "narHash": "sha256-T6oudJZTKD4JbMydet1ctY2MAcESEK8pVFGHa4uamy8=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "270e99600d656f47b49c1542b14c544a454c0132", + "type": "github" + }, + "original": { + "id": "nixpkgs", + "type": "indirect" + } + }, + "nixpkgs-mypy-0-740": { + "flake": false, + "locked": { + "lastModified": 1608136118, + "narHash": "sha256-yvnnBQrVsrWkRAEZDM3LiAVPSKBgk9ZgiSaa6NYmJX4=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "929768261a3ede470eafb58d5b819e1a848aa8bf", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-20.03", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-re2c-1-0-3": { + "flake": false, + "locked": { + "lastModified": 1600952148, + "narHash": "sha256-GUKHrnng33luc6mUT3rDnZ3Hm+4MMEJpEchRIAQx7JQ=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "75f4ba05c63be3f147bcc2f7bd4ba1f029cedcb1", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-19.09", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs", + "nixpkgs-mypy-0-740": "nixpkgs-mypy-0-740", + "nixpkgs-re2c-1-0-3": "nixpkgs-re2c-1-0-3" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000000..b8d2eb6279 --- /dev/null +++ b/flake.nix @@ -0,0 +1,152 @@ +# TODO(akavel): also create default.nix for compatibility with "classic nix" +# TODO(akavel): merge back stuff from shell.nix - ideally into mkDerivation's shellHook attribute +# TODO(akavel): try using `nixfmt`? + +{ + description = "A new Unix shell. Our upgrade path from bash to a better language and runtime."; + + inputs.nixpkgs-re2c-1-0-3 = { + url = github:NixOS/nixpkgs/nixos-19.09; + flake = false; # see: https://nixos.wiki/wiki/Flakes#Flake_schema + }; + + inputs.nixpkgs-mypy-0-740 = { + url = github:NixOS/nixpkgs/nixos-20.03; + flake = false; # see: https://nixos.wiki/wiki/Flakes#Flake_schema + }; + + outputs = { self, nixpkgs, nixpkgs-re2c-1-0-3, nixpkgs-mypy-0-740 }: { + + # defaultPackage.x86_64-linux = self.packages.x86_64-linux.oil; + # defaultPackage.x86_64-linux = self.packages.x86_64-linux.oil-tarball; + defaultPackage.x86_64-linux = self.packages.x86_64-linux.toil-run-cpp; + + # TODO(akavel): oil? or oil-cpp? or osh? or "all"? or...? + packages.x86_64-linux.toil-run-cpp = + with nixpkgs.legacyPackages.x86_64-linux; + let + selfpkgs = self.packages.x86_64-linux; + re2c-1-0-3 = (import nixpkgs-re2c-1-0-3 { system = "x86_64-linux"; }).pkgs.re2c; + mypy-0-730 = (import nixpkgs-mypy-0-740 { system = "x86_64-linux"; }).pkgs.mypy.overrideDerivation (old: { + version = "0.730"; + src = python3Packages.fetchPypi { + pname = "mypy"; + version = "0.730"; + sha256 = "0ygqviby0i4i3k2mlnr08f07dxvkh5ncl17m14bg4w07x128k9s2"; + }; + }); + in stdenv.mkDerivation { + name = "oil"; # FIXME(akavel): correct or not? + version = "dev"; + src = self; + # FIXME(akavel): nativeBuildInputs? or buildInputs? always confuse them :( + nativeBuildInputs = [ + git # TODO(akavel): try to make it unnecessary + python2 + readline + re2c-1-0-3 + mypy-0-730 + # FIXME(akavel): do we need separate mypy above with the line below? + (python3.withPackages (ps: with ps; [ mypy-0-730 ])) + ]; + patches = [ + ./nix-yajl.patch + # TODO(akavel): according to current flakes docs, we could recover timestamp (via lastModifiedDate or lastModified) and hash (via rev), + # see: https://github.com/NixOS/nix/blob/0df69d96e02ce4c9e17bd33333c5d78313341dd3/src/nix/flake.md#flake-format + ./nix-no-git.patch + ]; + postPatch = + # TODO(akavel): is the patchShebangs command below overeager? + '' + patchShebangs */*.{sh,py} + patchShebangs mycpp/*/*.{sh,py} + '' + # Without this, `pushd py-yajl` fails in some scripts + + '' + ln -s ${selfpkgs.py-yajl.src} py-yajl + '' + # Equivalent to patched-out yajl() function in build/dev.sh + + '' + ln -s ${selfpkgs.py-yajl}/lib/python2.7/site-packages/yajl.so . + '' + # Disarm `test/cpp-unit.sh deps`, provide re2c via Nix instead + + '' + mkdir -p _deps + ln -s ${re2c-1-0-3} _deps/re2c-1.0.3 + '' + # Patch out mycpp-clone and mycpp-deps, provide mycpp via Nix instead + # TODO(akavel): can we somehow make things smarter so that we don't have to do below block? + # FIXME(akavel): also, shouldn't hardcode python3.8 here + # FIXME(akavel): also, the mypy below should probably be mypy-0-730 !!!!! + + '' + mkdir -p _clone + ln -s ${mypy}/lib/python3.8/site-packages _clone/mypy + sed -i \ + -e '/^mycpp-clone /d' \ + -e '/^mycpp-deps /d' \ + services/toil-worker.sh + '' + # 'activate' is not necessary, because Nix effectively replaces virtualenv. + # TODO(akavel): can we make below PYTHONPATH tweaks unnecessary? + + '' + grep -rl '_tmp/mycpp-venv/bin/activate' {build,mycpp}/*.sh | + xargs -n1 sed -i \ + -e 's|source _tmp/mycpp-venv/bin/activate|true "PATCHED OUT _tmp/mycpp-venv/bin/activate/"|' \ + -e 's|\bPYTHONPATH=$MYPY_REPO |PYTHONPATH=$_NIXPYTHONPATH |' + export _NIXPYTHONPATH=$PYTHONPATH + '' + # NOTE(akavel): https://mypy.readthedocs.io/en/latest/running_mypy.html#mapping-file-paths-to-modules + # Otherwise, mypy gets confused w.r.t. module paths. (Why is this file even here?) + + '' + rm __init__.py + ''; + buildPhase = '' + runHook preBuild + + services/toil-worker.sh run-cpp + + runHook postBuild + ''; + # FIXME(akavel): what we do in installPhase? + installPhase = '' + mkdir -p $out + tar c -C _tmp toil | tar x -C $out/ + ''; + }; + + packages.x86_64-linux.py-yajl = + with nixpkgs.legacyPackages.x86_64-linux; + let + selfpkgs = self.packages.x86_64-linux; + in pkgs.python27Packages.buildPythonPackage { + pname = "py-yajl"; + # pname = "py-yajl-oil"; + version = "dev"; + src = pkgs.fetchFromGitHub { + # FIXME(akavel): source package is not os-dependent; where to put it in flake? + owner = "oilshell"; + repo = "py-yajl"; + rev = "eb561e9aea6e88095d66abcc3990f2ee1f5339df"; + sha256 = "09piyj7rmhgpqm84ynhdkywmpfshckyiphwhrs6k5gnbm3m356p3"; + }; + doCheck = false; # skip tests + nativeBuildInputs = [ pkgs.git ]; + preBuild = '' + rmdir yajl + ln -s ${selfpkgs.yajl.src} yajl + ''; + }; + + packages.x86_64-linux.yajl = + with nixpkgs.legacyPackages.x86_64-linux; + { + src = pkgs.fetchFromGitHub { + # FIXME(akavel): source package is not os-dependent; where to put it in flake? + owner = "lloyd"; + repo = "yajl"; + rev = "5e3a7856e643b4d6410ddc3f84bc2f38174f2872"; + sha256 = "1s4w2938s12ximaqc8v3k3knmxpi1l9j1gb5q1n3cwamzhn00jfh"; + }; + }; + }; +} diff --git a/nix-no-git.patch b/nix-no-git.patch new file mode 100644 index 0000000000..d26111a14d --- /dev/null +++ b/nix-no-git.patch @@ -0,0 +1,18 @@ +--- a/services/toil-worker.sh ++++ b/services/toil-worker.sh +@@ -299,10 +299,11 @@ save-metadata() { + # variable, but sr.ht doesn't. + # git rev-parse --abbrev-ref HEAD > $meta_dir/git-branch.txt + +- git log -n 1 --pretty='format:%H' > $meta_dir/commit-hash.txt +- # ISO 8601 format0 +- git log -n 1 --pretty='format:%aI' > $meta_dir/commit-date.txt +- git log -n 1 --pretty='format:%s' > $meta_dir/commit-line.txt # "subject" ++ echo GIT METADATA NOT YET AVAILABLE IN NIX > $meta_dir/commit-line.txt # "subject" ++ # git log -n 1 --pretty='format:%H' > $meta_dir/commit-hash.txt ++ # # ISO 8601 format0 ++ # git log -n 1 --pretty='format:%aI' > $meta_dir/commit-date.txt ++ # git log -n 1 --pretty='format:%s' > $meta_dir/commit-line.txt # "subject" + } + + job-main() { diff --git a/nix-yajl.patch b/nix-yajl.patch new file mode 100644 index 0000000000..41cf483502 --- /dev/null +++ b/nix-yajl.patch @@ -0,0 +1,33 @@ +--- a/build/dev.sh 2020-12-31 20:07:47.138174527 +0100 ++++ b/build/dev.sh 2020-12-31 23:12:33.464939155 +0100 +@@ -236,24 +236,7 @@ + } + + yajl() { +- ### Build and test yajl binding (depends on submodule) +- +- log '' +- log "py-yajl/setup.py -> yajl.so" +- +- pushd py-yajl >/dev/null +- python2 setup.py --quiet build_ext --inplace +- +- # DISABLED. It causes a lot of spew. And yajl will not make it into oil-native. +- # Adapted from py-yajl/runtests.sh +- # python2 tests/unit.py +- +- # Hm this test doesn't make any assertions. +- zcat test_data/issue_11.gz | python2 tests/issue_11.py >/dev/null +- popd >/dev/null +- +- # Link it in the repo root +- ln -s -f py-yajl/yajl.so . ++ log 'PATCHED OUT yajl()' + } + + clean() { +@@ -344,3 +327,4 @@ + fi + + "$@" ++