Page MenuHomeSoftware Heritage
Paste P375

nix-shell: swh-environment with nix and virtualenv
ActivePublic

Authored by ardumont on Mar 23 2019, 11:28 PM.
as of now, swh-environment within nix:
```
$ nix-shell swh.nix
```
The first time, nix will install all the necessary runtime tools.
Then create a new virtualenv "swh".
Then install the necessary swh tools.
Next time, it will connect to the virtualenv.
with $SWH_ENVIRONMENT_HOME/swh.nix as:
```
with import <nixpkgs> {};
let sources = import ./nix/sources.nix;
pkgs = (import sources.nixpkgs {});
ppackages = pkgs.python3Packages;
in stdenv.mkDerivation {
name = "swhEnv";
buildInputs = [
# required packages for virtualenvwrapper and pip to work
# python3Full
# ppackages.virtualenv
ppackages.virtualenvwrapper
ppackages.ipython
# runtime dependencies needed for the python modules pip will try
# to install. In this particular environment the python modules
# listed in the accumulated requirements.txt require the following
# packages to be installed locally in order to compile any binary
# extensions they may require.
openssl
libffi
libzip
stdenv
zlib
pkgs.docker
pkgs.systemd.lib systemd ppackages.systemd
# subvertpy needs this
apr aprutil pkgs.subversion.dev
# web (not enough)
pkgs.libxml2 python3Packages.lxml
postgresql_11
cassandra
apacheKafka
];
src = null;
# Necessary environment variable to ease build within the pip environment
PKG_CONFIG_PATH = "${pkgs.systemd.lib}/lib";
SVN_PREFIX = "${pkgs.subversion.dev}";
# LC_ALL = "C.UTF-8";
# LC_CTYPE = "C.UTF-8";
# LANG = "C.UTF-8";
shellHook =''
# set SOURCE_DATE_EPOCH so that we can use python wheels
SOURCE_DATE_EPOCH=$(date +%s)
source ${ppackages.virtualenvwrapper}/bin/virtualenvwrapper.sh
export ENV_NAME=swh2
export ENV_FOLDER=$HOME/.virtualenvs/$ENV_NAME/
install-or-update-env() {
echo "($ENV_NAME) $ENV_FOLDER: Install tools..."
pip install pytest tox pifpaf flake8 codespell pre-commit pdbpp
echo "($ENV_NAME) $ENV_FOLDER: Install pip swh packages..."
pip install $( $SWH_ENVIRONMENT_HOME/bin/pip-swh-packages --with-testing )
}
if [ ! -d $ENV_FOLDER ]; then
echo "Creating virtualenv $ENV_NAME at $ENV_FOLDER"
mkvirtualenv -p ${python3}/bin/python -a $SWH_ENVIRONMENT_HOME $ENV_NAME
install-or-update-env
echo "Creating virtualenv $ENV_NAME at $ENV_FOLDER. Done."
else
echo "Connecting to virtualenv $ENV_NAME at $ENV_FOLDER"
workon $ENV_NAME
fi
'';
}
```
Then to update the environment from within:
```
(swh2) $ install-or-update-env
```
nix/sources.json (fixed channel version):
```
{
"niv": {
"branch": "master",
"description": "Easy dependency management for Nix projects",
"homepage": "https://github.com/nmattia/niv",
"owner": "nmattia",
"repo": "niv",
"rev": "98c74a80934123cb4c3bf3314567f67311eb711a",
"sha256": "1w8n54hapd4x9f1am33icvngkqns7m3hl9yair38yqq08ffwg0kn",
"type": "tarball",
"url": "https://github.com/nmattia/niv/archive/98c74a80934123cb4c3bf3314567f67311eb711a.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
},
"nixpkgs": {
"branch": "nixos-19.09",
"description": "A read-only mirror of NixOS/nixpkgs tracking the released channels. Send issues and PRs to",
"homepage": "https://github.com/NixOS/nixpkgs",
"owner": "NixOS",
"repo": "nixpkgs-channels",
"rev": "68d2f8325c386fcb0ddbb466a91e053d2c09ddd2",
"sha256": "1kl24b2qsjcy8gllddkjxy5zrqdzrjzhydwi3lbncan0560f8aa8",
"type": "tarball",
"url": "https://github.com/NixOS/nixpkgs-channels/archive/68d2f8325c386fcb0ddbb466a91e053d2c09ddd2.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
}
}
```
nix/sources.nix:
```
# This file has been generated by Niv.
let
#
# The fetchers. fetch_<type> fetches specs of type <type>.
#
fetch_file = spec:
if spec.builtin or true then
builtins_fetchurl { inherit (spec) url sha256; }
else
pkgs.fetchurl { inherit (spec) url sha256; };
fetch_tarball = spec:
if spec.builtin or true then
builtins_fetchTarball { inherit (spec) url sha256; }
else
pkgs.fetchzip { inherit (spec) url sha256; };
fetch_git = spec:
builtins.fetchGit { url = spec.repo; inherit (spec) rev ref; };
fetch_builtin-tarball = spec:
builtins.trace
''
WARNING:
The niv type "builtin-tarball" will soon be deprecated. You should
instead use `builtin = true`.
$ niv modify <package> -a type=tarball -a builtin=true
''
builtins_fetchTarball { inherit (spec) url sha256; };
fetch_builtin-url = spec:
builtins.trace
''
WARNING:
The niv type "builtin-url" will soon be deprecated. You should
instead use `builtin = true`.
$ niv modify <package> -a type=file -a builtin=true
''
(builtins_fetchurl { inherit (spec) url sha256; });
#
# The sources to fetch.
#
sources = builtins.fromJSON (builtins.readFile ./sources.json);
#
# Various helpers
#
# The set of packages used when specs are fetched using non-builtins.
pkgs =
if hasNixpkgsPath
then
if hasThisAsNixpkgsPath
then import (builtins_fetchTarball { inherit (sources_nixpkgs) url sha256; }) {}
else import <nixpkgs> {}
else
import (builtins_fetchTarball { inherit (sources_nixpkgs) url sha256; }) {};
sources_nixpkgs =
if builtins.hasAttr "nixpkgs" sources
then sources.nixpkgs
else abort
''
Please specify either <nixpkgs> (through -I or NIX_PATH=nixpkgs=...) or
add a package called "nixpkgs" to your sources.json.
'';
hasNixpkgsPath = (builtins.tryEval <nixpkgs>).success;
hasThisAsNixpkgsPath =
(builtins.tryEval <nixpkgs>).success && <nixpkgs> == ./.;
# The actual fetching function.
fetch = name: spec:
if ! builtins.hasAttr "type" spec then
abort "ERROR: niv spec ${name} does not have a 'type' attribute"
else if spec.type == "file" then fetch_file spec
else if spec.type == "tarball" then fetch_tarball spec
else if spec.type == "git" then fetch_git spec
else if spec.type == "builtin-tarball" then fetch_builtin-tarball spec
else if spec.type == "builtin-url" then fetch_builtin-url spec
else
abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}";
# Ports of functions for older nix versions
# a Nix version of mapAttrs if the built-in doesn't exist
mapAttrs = builtins.mapAttrs or (
f: set: with builtins;
listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set))
);
# fetchTarball version that is compatible between all the versions of Nix
builtins_fetchTarball = { url, sha256 }@attrs:
let
inherit (builtins) lessThan nixVersion fetchTarball;
in
if lessThan nixVersion "1.12" then
fetchTarball { inherit url; }
else
fetchTarball attrs;
# fetchurl version that is compatible between all the versions of Nix
builtins_fetchurl = { url, sha256 }@attrs:
let
inherit (builtins) lessThan nixVersion fetchurl;
in
if lessThan nixVersion "1.12" then
fetchurl { inherit url; }
else
fetchurl attrs;
in
mapAttrs (
name: spec:
if builtins.hasAttr "outPath" spec
then abort
"The values in sources.json should not have an 'outPath' attribute"
else
spec // { outPath = fetch name spec; }
) sources
```
Generated code from the niv tool which allows to fix the derivation sources (reproducibility) [1]
[1] https://github.com/nmattia/niv

Event Timeline

ardumont created this paste.Mar 23 2019, 11:28 PM
ardumont edited the content of this paste. (Show Details)Mar 24 2019, 12:06 PM
ardumont edited the content of this paste. (Show Details)Mar 24 2019, 12:24 PM
ardumont edited the content of this paste. (Show Details)Mar 24 2019, 12:27 PM
ardumont edited the content of this paste. (Show Details)Apr 6 2019, 6:35 PM
ardumont changed the title of this paste from swh-environment within nix to nix-shell: swh-environment with nix and virtualenv.Mar 13 2020, 7:48 PM
ardumont edited the content of this paste. (Show Details)
ardumont edited the content of this paste. (Show Details)Mar 13 2020, 8:00 PM
ardumont edited the content of this paste. (Show Details)Mar 14 2020, 10:34 AM
ardumont edited the content of this paste. (Show Details)Mar 14 2020, 7:54 PM