From 9076f32d194b24f80078ad1166beb9904a42e0b9 Mon Sep 17 00:00:00 2001 From: Oleg Lebedev Date: Fri, 5 Jan 2024 10:13:45 +1100 Subject: [PATCH 1/4] pin nixpkgs --- default.nix | 19 +++++++++++-------- nix/sources.json | 12 ++++++++++++ shell.nix | 11 ++++++++--- 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/default.nix b/default.nix index eec3e16..633cb2f 100644 --- a/default.nix +++ b/default.nix @@ -1,17 +1,20 @@ -{ pkgs ? import { }, nix ? pkgs.nix }: +let sources = import ./nix/sources.nix; +in { pkgs ? import sources.nixpkgs { } }: + let - sources = import ./nix/sources.nix; naersk = pkgs.callPackage sources.naersk { }; gitignoreSource = (pkgs.callPackage sources.gitignore { }).gitignoreSource; blake3-src = sources.BLAKE3; -in (naersk.buildPackage { +in +(naersk.buildPackage { root = gitignoreSource ./.; - buildInputs = [ pkgs.openssl nix pkgs.ronn ]; + buildInputs = [ pkgs.openssl pkgs.nix pkgs.ronn ]; }).overrideAttrs (attrs: { - CNS_GIT_COMMIT = if builtins.pathExists ./.git then - pkgs.lib.commitIdFromGitRepo ./.git - else - "next"; + CNS_GIT_COMMIT = + if builtins.pathExists ./.git then + pkgs.lib.commitIdFromGitRepo ./.git + else + "next"; BLAKE3_CSRC = "${blake3-src}/c"; postBuild = "make -f nix/Makefile post-build"; postInstall = "make -f nix/Makefile post-install"; diff --git a/nix/sources.json b/nix/sources.json index a7139a3..6472101 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -34,5 +34,17 @@ "type": "tarball", "url": "https://github.com/nmattia/naersk/archive/88cd22380154a2c36799fe8098888f0f59861a15.tar.gz", "url_template": "https://github.com///archive/.tar.gz" + }, + "nixpkgs": { + "branch": "nixpkgs-unstable", + "description": "Nix Packages collection & NixOS", + "homepage": "", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "63143ac2c9186be6d9da6035fa22620018c85932", + "sha256": "1qvr6c7sisnzvcnyidllsv0bz0xsjfw3cg5s090y3az6sgrrlss0", + "type": "tarball", + "url": "https://github.com/NixOS/nixpkgs/archive/63143ac2c9186be6d9da6035fa22620018c85932.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" } } diff --git a/shell.nix b/shell.nix index 4145251..4057155 100644 --- a/shell.nix +++ b/shell.nix @@ -1,6 +1,11 @@ -{ pkgs ? import { } }: -let main = import ./default.nix { inherit pkgs; }; -in with pkgs; +let sources = import ./nix/sources.nix; +in { pkgs ? import sources.nixpkgs { } }: + +with pkgs; + +let + main = import ./default.nix { inherit pkgs; }; +in mkShell { buildInputs = main.buildInputs ++ main.nativeBuildInputs ++ [ cargo-edit From 73b2f5e99c5be2ac7286f06dec0a0b0e115e4ab1 Mon Sep 17 00:00:00 2001 From: Oleg Lebedev Date: Fri, 5 Jan 2024 10:28:17 +1100 Subject: [PATCH 2/4] clean up the default.nix a bit --- default.nix | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/default.nix b/default.nix index 633cb2f..25c7af0 100644 --- a/default.nix +++ b/default.nix @@ -5,17 +5,30 @@ let naersk = pkgs.callPackage sources.naersk { }; gitignoreSource = (pkgs.callPackage sources.gitignore { }).gitignoreSource; blake3-src = sources.BLAKE3; + + # The main cached-nix-shell package. It's subject for override (see below) of the + # underlying derivation attributes so the build works correctly. + package = naersk.buildPackage { + root = gitignoreSource ./.; + buildInputs = with pkgs; [ + openssl + nix + ronn + ]; + }; + # Final overrides. + package' = package.overrideAttrs (_: { + CNS_GIT_COMMIT = + if builtins.pathExists ./.git + then pkgs.lib.commitIdFromGitRepo ./.git + else "next"; + BLAKE3_CSRC = "${blake3-src}/c"; + postBuild = '' + make -f nix/Makefile post-build + ''; + postInstall = '' + make -f nix/Makefile post-install + ''; + }); in -(naersk.buildPackage { - root = gitignoreSource ./.; - buildInputs = [ pkgs.openssl pkgs.nix pkgs.ronn ]; -}).overrideAttrs (attrs: { - CNS_GIT_COMMIT = - if builtins.pathExists ./.git then - pkgs.lib.commitIdFromGitRepo ./.git - else - "next"; - BLAKE3_CSRC = "${blake3-src}/c"; - postBuild = "make -f nix/Makefile post-build"; - postInstall = "make -f nix/Makefile post-install"; -}) +package' From cbfb0a58265b324db983bc7e2e1ef8167dda2b58 Mon Sep 17 00:00:00 2001 From: Oleg Lebedev Date: Fri, 5 Jan 2024 11:37:42 +1100 Subject: [PATCH 3/4] feat: substitute executables at the build time --- Cargo.lock | 12 ---------- Cargo.toml | 3 --- build.rs | 32 +++++++++++++------------ default.nix | 23 +++++++++--------- shell.nix | 8 +++---- src/args.rs | 9 ++----- src/main.rs | 66 +++++++++------------------------------------------- src/trace.rs | 4 ++-- 8 files changed, 47 insertions(+), 110 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index eab8f86..87d1935 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -124,7 +124,6 @@ dependencies = [ "serde_json", "tempfile", "ufcs", - "which", "xdg", ] @@ -678,17 +677,6 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" -[[package]] -name = "which" -version = "4.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" -dependencies = [ - "either", - "libc", - "once_cell", -] - [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index 90a444f..0b5fdca 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,9 +5,6 @@ authors = ["Albert Safin "] license = "Unlicense OR MIT" edition = "2018" -[build-dependencies] -which = "4.4.0" - [dependencies] blake3 = "1.3.3" bytelines = "2.4.0" diff --git a/build.rs b/build.rs index a6f200d..fb3e64e 100644 --- a/build.rs +++ b/build.rs @@ -3,29 +3,34 @@ use std::path::Path; use std::process::Command; fn main() { + // Necessary env var to substitute in the final artifact depends on, regardless of the + // build context; either in nix-shell or a basic nix build. + println!( + "cargo:rustc-env=CNS_ESSENTIALS={}", + var("ESSENTIALS").expect("Expect to the ESSENTIALS env var to be set.") + ); + println!( + "cargo:rustc-env=CNS_BASH={}", + var("BASH").expect("Expect to the BASH env var to be set.") + ); + println!( + "cargo:rustc-env=CNS_NIX={}/", + var("NIX_BIN").expect("Expect to the NIX_BIN env var to be set.") + ); + if var_os("CNS_IN_NIX_SHELL").is_none() { // Release build triggered by nix-build. Use paths relative to $out. let out = var("out").unwrap(); println!("cargo:rustc-env=CNS_TRACE_NIX_SO={out}/lib/trace-nix.so"); println!("cargo:rustc-env=CNS_VAR_EMPTY={out}/var/empty"); println!( + // This file is moved to /share/cached-nix-shell in the ./nix/Makefile#post-install "cargo:rustc-env=CNS_RCFILE={out}/share/cached-nix-shell/rcfile.sh" ); println!( + // This directory is created in the ./nix/Makefile#post-install "cargo:rustc-env=CNS_WRAP_PATH={out}/libexec/cached-nix-shell" ); - - // Use pinned nix and nix-shell binaries. - println!( - "cargo:rustc-env=CNS_NIX={}/", - which::which("nix") - .expect("command not found: nix") - .parent() - .unwrap() - .as_os_str() - .to_str() - .unwrap() - ); } else { // Developer build triggered by `nix-shell --run 'cargo build'`. // Use paths relative to the build directory. Additionally, place @@ -59,8 +64,5 @@ fn main() { ) .unwrap(); println!("cargo:rustc-env=CNS_WRAP_PATH={out_dir}/wrapper"); - - // Use nix and nix-shell from $PATH at runtime. - println!("cargo:rustc-env=CNS_NIX="); } } diff --git a/default.nix b/default.nix index 25c7af0..53267cd 100644 --- a/default.nix +++ b/default.nix @@ -6,22 +6,24 @@ let gitignoreSource = (pkgs.callPackage sources.gitignore { }).gitignoreSource; blake3-src = sources.BLAKE3; + ESSENTIALS = with pkgs; + lib.makeBinPath [ bashInteractive coreutils nix gitMinimal gnutar gzip ]; + BASH = "${pkgs.bashInteractive}/bin/bash"; + NIX_BIN = "${pkgs.nix}/bin"; + # The main cached-nix-shell package. It's subject for override (see below) of the # underlying derivation attributes so the build works correctly. package = naersk.buildPackage { root = gitignoreSource ./.; - buildInputs = with pkgs; [ - openssl - nix - ronn - ]; + buildInputs = with pkgs; [ openssl nix ronn ]; }; # Final overrides. package' = package.overrideAttrs (_: { - CNS_GIT_COMMIT = - if builtins.pathExists ./.git - then pkgs.lib.commitIdFromGitRepo ./.git - else "next"; + inherit ESSENTIALS BASH NIX_BIN; + CNS_GIT_COMMIT = if builtins.pathExists ./.git then + pkgs.lib.commitIdFromGitRepo ./.git + else + "next"; BLAKE3_CSRC = "${blake3-src}/c"; postBuild = '' make -f nix/Makefile post-build @@ -30,5 +32,4 @@ let make -f nix/Makefile post-install ''; }); -in -package' +in package' diff --git a/shell.nix b/shell.nix index 4057155..600de05 100644 --- a/shell.nix +++ b/shell.nix @@ -3,10 +3,8 @@ in { pkgs ? import sources.nixpkgs { } }: with pkgs; -let - main = import ./default.nix { inherit pkgs; }; -in -mkShell { +let main = import ./default.nix { inherit pkgs; }; +in mkShell { buildInputs = main.buildInputs ++ main.nativeBuildInputs ++ [ cargo-edit clippy @@ -20,7 +18,7 @@ mkShell { git openssh ]; - inherit (main) BLAKE3_CSRC; + inherit (main) ESSENTIALS BASH NIX_BIN BLAKE3_CSRC; CNS_IN_NIX_SHELL = "1"; RUST_SRC_PATH = "${pkgs.rust.packages.stable.rustPlatform.rustLibSrc}"; } diff --git a/src/args.rs b/src/args.rs index e3146ee..f52b2fd 100644 --- a/src/args.rs +++ b/src/args.rs @@ -10,6 +10,7 @@ //! compatible way, so it is appropriate to code this explicitly rather than use //! such libraries. +use log::info; use std::collections::VecDeque; use std::ffi::{OsStr, OsString}; use std::io::Write; @@ -17,7 +18,6 @@ use std::os::unix::ffi::OsStrExt; use std::os::unix::process::CommandExt; use std::process::{exit, Command}; use ufcs::Pipe; -use log::info; pub enum RunMode { /// no arg @@ -97,7 +97,7 @@ impl Args { packages_or_expr: false, pure: false, include_nix_path: Vec::new(), - interpreter: OsString::from("bash"), + interpreter: env!("CNS_BASH").into(), run: RunMode::InteractiveShell, keep: Vec::new(), rest: Vec::new(), @@ -204,11 +204,6 @@ fn exit_version() { .map(|x| format!("-{x}")) .unwrap_or("".into()) ); - if env!("CNS_NIX").is_empty() { - info!("Using nix-shell from $PATH"); - } else { - info!("Using {}nix-shell", env!("CNS_NIX")); - } std::io::stdout().flush().unwrap(); Command::new(concat!(env!("CNS_NIX"), "nix-shell")) .arg("--version") diff --git a/src/main.rs b/src/main.rs index 6cd7bba..e3108f3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,10 +2,12 @@ use crate::args::Args; use crate::bash::is_literal_bash_string; use crate::path_clean::PathClean; use crate::trace::Trace; +use env_logger::{Builder, Env}; use itertools::{chain, Itertools}; +use log::{debug, error, info, warn}; use nix::unistd::{access, AccessFlags}; use once_cell::sync::Lazy; -use std::collections::{BTreeMap, HashSet}; +use std::collections::BTreeMap; use std::env::current_dir; use std::ffi::{OsStr, OsString}; use std::fs::{read, read_link, File}; @@ -19,8 +21,6 @@ use std::process::{exit, Command, Stdio}; use std::time::Instant; use tempfile::NamedTempFile; use ufcs::Pipe; -use log::{debug, error, warn, info}; -use env_logger::{Builder, Env}; mod args; mod bash; @@ -109,50 +109,6 @@ struct NixShellOutput { drv: String, } -fn minimal_essential_path() -> OsString { - let required_binaries = ["tar", "gzip", "git", "nix-shell", "rm"]; - - fn which_dir(binary: &&str) -> Option { - std::env::var_os("PATH") - .as_ref() - .unwrap() - .pipe(std::env::split_paths) - .find(|dir| { - if access(&dir.join(binary), AccessFlags::X_OK).is_err() { - return false; - } - - if binary == &"nix-shell" { - // Ignore our fake nix-shell. - return !dir - .join(binary) - .canonicalize() - .ok() - .and_then(|x| x.file_name().map(|x| x.to_os_string())) - .map(|x| x == "cached-nix-shell") - .unwrap_or(true); - } - - true - }) - } - - let required_paths = required_binaries - .iter() - .filter_map(which_dir) - .collect::>(); - - // We can't just join_paths(required_paths) -- we need to preserve order - std::env::var_os("PATH") - .as_ref() - .unwrap() - .pipe(std::env::split_paths) - .filter(|path_item| required_paths.contains(path_item)) - .unique() - .pipe(std::env::join_paths) - .unwrap() -} - fn absolute_dirname(script_fname: &OsStr) -> PathBuf { Path::new(&script_fname) .parent() @@ -209,7 +165,7 @@ fn args_to_inp(pwd: PathBuf, x: &Args) -> NixShellInput { args.push(var.clone()); } } - clean_env.insert(OsString::from("PATH"), minimal_essential_path()); + clean_env.insert(OsString::from("PATH"), env!("CNS_ESSENTIALS").into()); clean_env }; @@ -217,6 +173,8 @@ fn args_to_inp(pwd: PathBuf, x: &Args) -> NixShellInput { args.push(OsString::from("--")); args.extend(x.rest.clone()); + debug!("env: {:?}", &env); + NixShellInput { pwd, env, @@ -307,9 +265,7 @@ fn run_nix_shell(inp: &NixShellInput) -> NixShellOutput { stderr.extend(exec.stderr); } if !exec.status.success() { - error!( - "failed to execute nix show-derivation" - ); + error!("failed to execute nix show-derivation"); let _ = std::io::stderr().write_all(&stderr); exit(1); } @@ -350,7 +306,7 @@ fn run_script( exec_string.push("exec "); exec_string.push(nix_shell_args.interpreter); exec_string.push(r#" "$@""#); - Command::new("bash") + Command::new(env!("CNS_BASH")) .arg("-c") .arg(exec_string) .arg("cached-nix-shell-bash") // corresponds to "$0" inside '-i' @@ -428,12 +384,12 @@ fn run_from_args(args: Vec) { args::RunMode::InteractiveShell => { let mut args = vec!["--rcfile".into(), env!("CNS_RCFILE").into()]; args.append(build_bash_options(&env).as_mut()); - ("bash".into(), args) + (env!("CNS_BASH").into(), args) } args::RunMode::Shell(cmd) => { let mut args = build_bash_options(&env); args.extend_from_slice(&["-c".into(), cmd]); - ("bash".into(), args) + (env!("CNS_BASH").into(), args) } args::RunMode::Exec(cmd, cmd_args) => (cmd, cmd_args), }; @@ -636,7 +592,7 @@ fn wrap(cmd: Vec) { .args(&cmd[1..]) .env("PATH", OsStr::from_bytes(&new_path)) .exec(); - error!("couldn't run: {exec}"); + error!("couldn't execute: {exec}"); exit(1); } diff --git a/src/trace.rs b/src/trace.rs index 7009c55..643e7f3 100644 --- a/src/trace.rs +++ b/src/trace.rs @@ -1,10 +1,10 @@ use itertools::Itertools; +use log::info; use std::collections::BTreeMap; use std::ffi::{OsStr, OsString}; use std::fs::{read, read_dir, read_link, symlink_metadata}; use std::io::ErrorKind; use std::os::unix::ffi::OsStrExt; -use log::error; /// Output of trace-nix.so, sorted and deduplicated. pub struct Trace { @@ -81,7 +81,7 @@ fn check_item_updated(k: &[u8], v: &[u8]) -> bool { }; if res.as_bytes() != v { - error!( + info!( "{:?}: expected {:?}, got {:?}", fname, OsStr::from_bytes(v), From e87b9e4d8d968adbda84afba0abaf388e75718ca Mon Sep 17 00:00:00 2001 From: Oleg Lebedev Date: Fri, 5 Jan 2024 14:19:42 +1100 Subject: [PATCH 4/4] set minimum supported Nix version to 2.4 & make the build pass for non-nix environments --- build.rs | 34 ++++++++++++++++++++++------------ default.nix | 11 +++++++++-- shell.nix | 2 +- src/main.rs | 23 ++++++----------------- 4 files changed, 38 insertions(+), 32 deletions(-) diff --git a/build.rs b/build.rs index fb3e64e..cb3601e 100644 --- a/build.rs +++ b/build.rs @@ -5,18 +5,28 @@ use std::process::Command; fn main() { // Necessary env var to substitute in the final artifact depends on, regardless of the // build context; either in nix-shell or a basic nix build. - println!( - "cargo:rustc-env=CNS_ESSENTIALS={}", - var("ESSENTIALS").expect("Expect to the ESSENTIALS env var to be set.") - ); - println!( - "cargo:rustc-env=CNS_BASH={}", - var("BASH").expect("Expect to the BASH env var to be set.") - ); - println!( - "cargo:rustc-env=CNS_NIX={}/", - var("NIX_BIN").expect("Expect to the NIX_BIN env var to be set.") - ); + + // We need to support non-nix environment anyways, ecpesially for VSCode and rust-analyser that + // is being run by a VSCode extension and it doesn't have access to nix-shell to do a proper build. + if var("IN_NIX").is_err() || var("IN_NIX").unwrap() != "1" { + println!("cargo:rustc-env=CNS_ESSENTIALS=/nix/var/nix/profiles/default/bin:/usr/local/bin:/usr/bin:/usr/sbin:/bin:/sbin:/opt/homebrew/bin:/opt/homebrew/sbin"); + println!("cargo:rustc-env=CNS_BASH=bash"); + println!("cargo:rustc-env=CNS_NIX="); + } else { + println!( + "cargo:rustc-env=CNS_ESSENTIALS={}", + var("ESSENTIALS") + .expect("Expect to the ESSENTIALS env var to be set.") + ); + println!( + "cargo:rustc-env=CNS_BASH={}", + var("BASH").expect("Expect to the BASH env var to be set.") + ); + println!( + "cargo:rustc-env=CNS_NIX={}/", + var("NIX_BIN").expect("Expect to the NIX_BIN env var to be set.") + ); + } if var_os("CNS_IN_NIX_SHELL").is_none() { // Release build triggered by nix-build. Use paths relative to $out. diff --git a/default.nix b/default.nix index 53267cd..a58630b 100644 --- a/default.nix +++ b/default.nix @@ -1,6 +1,12 @@ -let sources = import ./nix/sources.nix; +let + sources = import ./nix/sources.nix; + minNixVer = "2.4"; in { pkgs ? import sources.nixpkgs { } }: +assert pkgs.lib.assertMsg + (builtins.compareVersions minNixVer pkgs.nix.version != 1) + "The project doesn't support Nix of version ${pkgs.nix.version}. The minimum required version is ${minNixVer}."; + let naersk = pkgs.callPackage sources.naersk { }; gitignoreSource = (pkgs.callPackage sources.gitignore { }).gitignoreSource; @@ -10,6 +16,7 @@ let lib.makeBinPath [ bashInteractive coreutils nix gitMinimal gnutar gzip ]; BASH = "${pkgs.bashInteractive}/bin/bash"; NIX_BIN = "${pkgs.nix}/bin"; + IN_NIX = true; # The main cached-nix-shell package. It's subject for override (see below) of the # underlying derivation attributes so the build works correctly. @@ -19,7 +26,7 @@ let }; # Final overrides. package' = package.overrideAttrs (_: { - inherit ESSENTIALS BASH NIX_BIN; + inherit ESSENTIALS BASH NIX_BIN IN_NIX; CNS_GIT_COMMIT = if builtins.pathExists ./.git then pkgs.lib.commitIdFromGitRepo ./.git else diff --git a/shell.nix b/shell.nix index 600de05..982e261 100644 --- a/shell.nix +++ b/shell.nix @@ -18,7 +18,7 @@ in mkShell { git openssh ]; - inherit (main) ESSENTIALS BASH NIX_BIN BLAKE3_CSRC; + inherit (main) ESSENTIALS BASH NIX_BIN IN_NIX BLAKE3_CSRC; CNS_IN_NIX_SHELL = "1"; RUST_SRC_PATH = "${pkgs.rust.packages.stable.rustPlatform.rustLibSrc}"; } diff --git a/src/main.rs b/src/main.rs index e3108f3..11d6c0f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,7 +3,7 @@ use crate::bash::is_literal_bash_string; use crate::path_clean::PathClean; use crate::trace::Trace; use env_logger::{Builder, Env}; -use itertools::{chain, Itertools}; +use itertools::chain; use log::{debug, error, info, warn}; use nix::unistd::{access, AccessFlags}; use once_cell::sync::Lazy; @@ -245,28 +245,17 @@ fn run_nix_shell(inp: &NixShellInput) -> NixShellOutput { std::mem::drop(trace_file); let drv: String = { - // nix 2.3 - let mut exec = Command::new(concat!(env!("CNS_NIX"), "nix")) + // nix 2.4 + let exec = Command::new(concat!(env!("CNS_NIX"), "nix")) .arg("show-derivation") + .arg("--extra-experimental-features") + .arg("nix-command") .arg(env_out) .output() .expect("failed to execute nix show-derivation"); - let mut stderr = exec.stderr.clone(); - if !exec.status.success() { - // nix 2.4 - exec = Command::new(concat!(env!("CNS_NIX"), "nix")) - .arg("show-derivation") - .arg("--extra-experimental-features") - .arg("nix-command") - .arg(env_out) - .output() - .expect("failed to execute nix show-derivation"); - stderr.extend(b"\n"); - stderr.extend(exec.stderr); - } if !exec.status.success() { error!("failed to execute nix show-derivation"); - let _ = std::io::stderr().write_all(&stderr); + let _ = std::io::stderr().write_all(&exec.stderr); exit(1); }