From dc9988194d061277ebdb55d10fccc39ac3cd30b8 Mon Sep 17 00:00:00 2001 From: Connor Olding Date: Thu, 15 Feb 2024 16:59:39 -0800 Subject: [PATCH] automamba: update, add linux support, improve robustness --- automamba/automamba | 272 +++++++++++++++++++++++++++++++------------- 1 file changed, 190 insertions(+), 82 deletions(-) diff --git a/automamba/automamba b/automamba/automamba index b97be1a..dab3c4b 100644 --- a/automamba/automamba +++ b/automamba/automamba @@ -1,23 +1,37 @@ #!/usr/bin/env bash +unset CDPATH -home="${USERPROFILE}" +if [ -d /C ]; then # Windows +home="$USERPROFILE" # NOTE: checked for existence later instead of here. downloads="$home\\Downloads" base="$home\\mamba" -channel=main -#channel=conda-forge +else # Linux +home="$HOME" # NOTE: checked for existence later instead of here. +downloads="$home/src" +base="$home/mamba" +fi # micromamba stuff; doesn't affect python itself: -installer_version="0.23.3" -installer_sha256="d133dc61615b03fa953b6563c6d0f7362c5319c1ed164a4c1311b7f5e1868776" +installer_version="1.5.6" # remember to update both sha256s if you change this! installer_archive="micromamba-$installer_version-0.tar.bz2" -installer_remote="https://micromamba.snakepit.net/api/micromamba/win-64/$installer_version" + +releases="https://github.com/mamba-org/micromamba-releases/releases/download" +if [ -d /C ]; then # Windows +installer_sha256="e214c76c0ff1506c4b6f3ef31064250eb71f9c2698c108bfc29a5b31e3a8ab6f" +installer_remote="$releases/$installer_version-0/micromamba-win-64.tar.bz2" installer_path="Library/bin/micromamba.exe" +else # Linux +installer_sha256="efe462c7ffcae8b338c7dd7b168ce8d48cfc60b48ab991d02a035c3b8d73633c" +installer_remote="$releases/$installer_version-0/micromamba-linux-64.tar.bz2" +installer_path="bin/micromamba" +fi # remember not to use == when you mean = here: -python='python=3.10' +python='python=3.11' +channel=conda-forge main_packages=( - 'pip>=21.2.4' # python package manager + 'pip>=23.3.1' # python package manager ) # pip packages should be pinned with pure equality @@ -33,23 +47,65 @@ check_installer() { printf '%s %s\n' "$installer_sha256" "$1" | sha256sum -c - || return } -run_mamba() { - # TODO: infer System32 path by envvars. (it's looking for chcp.exe) - PATH="\ -C:\\Windows\\System32:\ -C:\\Windows\\" \ - "$fullbase\\Library\\bin\\micromamba.exe" --no-rc --no-env -r "$base" "$@" +mamba_paths() { + if [ -d /C ]; then # Windows + # TODO: infer System32 path by envvars. (it's looking for chcp.com) + printf %s \ + "$(cygpath -u "C:\\Windows\\System32")" \ + : \ + "$(cygpath -u "C:\\Windows\\")" \ + ; + else # Linux + printf %s \ + "/usr/local/sbin" \ + : \ + "/usr/local/bin" \ + : \ + "/usr/sbin" \ + : \ + "/usr/bin" \ + : \ + "/sbin" \ + : \ + "/bin" \ + ; + fi } -run_python() { - PATH="\ -$fullbase\\envs\\$env:\ -$fullbase\\envs\\$env\\Library\\mingw-w64\\bin:\ -$fullbase\\envs\\$env\\Library\\bin" \ - "$fullbase\\envs\\$env\\python.exe" "$@" +python_paths() { # relies on globals: fullbase, env + if [ -d /C ]; then # Windows + printf %s \ + "$(cygpath -u "$fullbase\\envs\\$env")" \ + : \ + "$(cygpath -u "$fullbase\\envs\\$env\\Library\\mingw-w64\\bin")" \ + : \ + "$(cygpath -u "$fullbase\\envs\\$env\\Library\\bin")" \ + ; + else # Linux + printf %s \ + "$(readlink -f "$fullbase/envs/$env/x86_64-conda-linux-gnu/bin")" \ + : \ + "$(readlink -f "$fullbase/envs/$env/bin")" \ + ; + fi + printf %s "${MOREPATH:+:}$MOREPATH" } -find_msvc() { +run_mamba() { # relies on globals: fullbase, base, env + local paths= + paths="$(mamba_paths)" || return + PATH="$paths" "$fullbase/$installer_path" --no-rc --no-env -r "$base" "$@" +} + +run_python() { # relies on globals: fullbase, env + local exe= paths= + [ -d /C ] && exe=python.exe || exe=python + paths="$(python_paths)" || return + PATH="$paths" "$exe" "$@" +} + +find_msvc() { # TODO: does this use any globals? besides SYSTEMDRIVE which is an envvar. + [ -d /C ] || { echo 'Internal error: attempted Windows routine on Linux!' >&2; exit 1; } [ -n "$SYSTEMDRIVE" ] || { echo '$SYSTEMDRIVE must be set.' >&2; exit 1; } dirs= libs= lib= includes= @@ -107,19 +163,27 @@ find_msvc() { #exit 1 } -install_msvc() { +install_msvc() { # TODO: document globals used. + [ -d /C ] || { echo 'Internal error: attempted Windows routine on Linux!' >&2; exit 1; } [ -n "$SYSTEMDRIVE" ] || { echo '$SYSTEMDRIVE must be set.' >&2; exit 1; } [ -n "$COMSPEC" ] || { echo '$COMSPEC must be set.' >&2; exit 1; } # find the latest version: curl -I 'https://aka.ms/vs/17/release/vs_buildtools.exe' - local uuid='05734053-383e-4b1a-9950-c7db8a55750d' - local hash='fbfc005ace3e6b4990e9a4be0fa09e7face1af5ee1f61035c64dbc16c407aeda' + local uuid='5bebe58c-9308-4a5b-9696-b6f84e90a32e' + local hash='d62702bf9e2bb2c8be1f85ec4b86e0426e42646d12ac5196c451574d22be148e' local exe='vs_BuildTools.exe' local installer="$downloads\\$exe" local url="https://download.visualstudio.microsoft.com/download/pr/$uuid/$hash/$exe" local profiles="Program Files (x86)" - [ -s "$installer" ] || curl -L "$url" -o "$installer" || return + mode=install + if [ "$1" = update ]; then + shift + mode=update + fi + + # always download the installer since it's only a couple megabytes. + curl -L "$url" -o "$installer" || return printf '%s %s\n' "$hash" "$installer" | sha256sum -c - || return cd "$SYSTEMDRIVE" || return @@ -131,7 +195,7 @@ install_msvc() { # TODO: use "update" instead of "install" when applicable. # despite the name, you can use the Windows 11 SDK (or at least this version of it) # to target as early as Windows 10 v1507. - "$installer" --quiet --wait --norestart --nocache install \ + "$installer" --quiet --wait --norestart --nocache "$mode" \ --add 'Microsoft.VisualStudio.Component.VC.Tools.x86.x64' \ --add 'Microsoft.VisualStudio.Component.Windows11SDK.22000' local ret=$? @@ -166,41 +230,44 @@ install_msvc() { [ "$code" = 0 ] || [ "$code" = 3010 ] || return 1 } -run_with_compiler() { - find_msvc || install_msvc \ - || { printf 'failed to install MSVC and a Windows SDK.\n' >&2; exit 1; } - find_msvc \ - || { printf 'failed to find a valid installation of MSVC.\n' >&2; exit 1; } +run_with_compiler() { # relies on globals: fullbase, env, working_dir + if [ -d /C ]; then # Windows + find_msvc || install_msvc \ + || { printf 'failed to install MSVC and a Windows SDK.\n' >&2; exit 1; } + find_msvc \ + || { printf 'failed to find a valid installation of MSVC.\n' >&2; exit 1; } + # globals should now exist: dirs, libs, lib, includes - cd "$working_dir" || return - PATH="\ -$fullbase\\envs\\$env:\ -$fullbase\\envs\\$env\\Library\\mingw-w64\\bin:\ -$fullbase\\envs\\$env\\Library\\bin:\ -$dirs" \ - LIB="$libs" \ - LIBPATH="$lib" \ - INCLUDE="$includes" \ - python.exe "$@" + ( + cd "$working_dir" || return + export CMAKE_GENERATOR="Visual Studio 17 2022" + MOREPATH="$dirs" LIB="$libs" LIBPATH="$lib" INCLUDE="$includes" \ + run_python "$@" + ) + + else # Linux + printf '%s\n' 'run_with_compiler: TODO: handle static gcc here!' >&2 + run_python "$@" + fi } run_pip() { run_with_compiler -m pip --disable-pip-version-check "$@" } -find_last_env() { +find_last_env() { # relies on globals: fullbase local arg="$1" e= oe= env="$(date -u '+%Y%m%d')" || exit n=1 - while [ -d "$fullbase\\envs\\$env-$n" ]; do + while [ -d "$fullbase/envs/$env-$n" ]; do let n++ done new_env="$env-$n" let n-- if [ $n = 0 ]; then env=_ - if [ -d "$fullbase\\envs" ]; then - for e in "$fullbase\\envs/"*; do # must use a forward slash for some reason + if [ -d "$fullbase/envs" ]; then + for e in "$fullbase/envs/"*; do oe="${e##*/}" #printf 'considering %s... compared to %s\n' "$oe" "$env" >&2 if [ -d "$e" ] && [ "${oe#20}" != "$oe" ] && [[ "$oe" > "$env" ]]; then @@ -223,35 +290,51 @@ find_last_env() { # end of functions, begin main: -working_dir="$PWD" +if [ -d /C ]; then # Windows + [ -n "$USERPROFILE" ] || { echo '$USERPROFILE must be set.' >&2; exit 1; } +else # Linux + [ -n "$HOME" ] || { echo '$HOME must be set.' >&2; exit 1; } +fi -[ -n "$USERPROFILE" ] || { echo '$USERPROFILE must be set.' >&2; exit 1; } -[ -d "$working_dir" ] || { echo 'wat?' >&2; exit 1; } +[ -d "$downloads" ] || mkdir "$downloads" || exit + +working_dir="$PWD" +[ -d "$working_dir" ] || { echo 'failed to determine working directory' >&2; exit 1; } [ -d "$base" ] || mkdir "$base" || exit -fullbase="$(cygpath -u "$base")" || exit -fullbase="$(readlink -f "$fullbase")" || exit -fullbase="$(cygpath -w "$fullbase")" || exit +if [ -d /C ]; then # Windows + fullbase="$(cygpath -u "$base")" || exit + fullbase="$(readlink -f "$fullbase")" || exit + fullbase="$(cygpath -w "$fullbase")" || exit +else # Linux + fullbase="$(readlink -f "$base")" || exit +fi version="unknown" -if [ -s "$fullbase\\$installer_path" ]; then +if [ -s "$fullbase/$installer_path" ]; then # TODO: do something meaningful if the execution itself fails. version="$(run_mamba --version)" fi if [ "$version" != "$installer_version" ]; then - archive="$downloads\\$installer_archive" + archive="$downloads/$installer_archive" if check_installer "$archive"; then : else - # slightly less safe: - # curl -LOJ "$installer_remote" + # slightly less safe: curl -LOJ "$installer_remote" curl -L "$installer_remote" -o "$archive" || exit check_installer "$archive" || exit fi - archive="$(cygpath -u "$archive")" || exit + ! [ -d /C ] || archive="$(cygpath -u "$archive")" || exit cd "$fullbase" || exit - tar jxf "$archive" "$installer_path" || exit + if ! which bzip2 >/dev/null 2>&1 && which python3 >/dev/null 2>&1; then + # ubuntu-server does not come with bzip2, but python3 can save us here. + # 'import tarfile,sys;_,a,b,c=sys.argv;tarfile.open(a,"r:bz2").extract(b,c) + python3 -c 'import tarfile,sys;_,a,b=sys.argv;tarfile.open(a,"r:bz2").extract(b)' \ + "$archive" "$installer_path" || exit + else + tar jxf "$archive" "$installer_path" || exit + fi cd "$working_dir" || exit fi @@ -259,8 +342,28 @@ fi [ "$1" = --mamba ] && dancing=1 || dancing=0 [ "$1" = --pip ] && compiling=1 || compiling=0 [ "$1" = --wheel ] && wheeling=1 || wheeling=0 +[ "$1" = --msvc ] && visualizing=1 || visualizing=0 -find_last_env new || exit +if [ "${AM_ENV:-__empty__}" = __empty__ ]; then + find_last_env new || exit +else + if [ $creating = 1 ]; then + : + elif [ ! -d "$fullbase/envs/$AM_ENV" ]; then + printf 'failed to find environment $AM_ENV: %s\n' "$AM_ENV" >&2 + sleep 3 #exit 1 + elif [ ! -s "$fullbase/envs/$AM_ENV/LICENSE_PYTHON.txt" ]; then + printf 'empty environment $AM_ENV: %s\n' "$AM_ENV" >&2 + sleep 3 #exit 1 + fi + env="$AM_ENV" +fi + +if [ $visualizing = 1 ]; then + [ $visualizing = 0 ] || shift || exit + install_msvc update + exit +fi if [ $dancing = 1 ]; then [ $dancing = 0 ] || shift || exit @@ -305,40 +408,45 @@ fi if [ "$env" = ERROR ] || [ $creating != 0 ]; then [ $creating = 0 ] || shift || exit - env="$new_env" + [ -z "$new_env" ] || env="$new_env" if [ $creating = 0 ]; then - printf 'WARNING: automatically creating a new environment: %s' "$env" >&2 + printf 'WARNING: automatically creating a new environment: %s\n' "$env" >&2 fi - run_mamba -yqn "$env" create -c "$channel" \ - "$python" "${main_packages[@]}" 2>&1 \ - | tee "$fullbase\\create.log" + _do_create() { + if [ $# = 0 ]; then + run_mamba -yqn "$env" create -c "$channel" "$python" "${main_packages[@]}" + else + run_mamba -yqn "$env" create "$@" + fi + } + + if [ $creating = 1 ]; then + _do_create "$@" 2>&1 | tee "$fullbase/create.log" + else + _do_create 2>&1 | tee "$fullbase/create.log" + fi res=${PIPESTATUS[0]} - if [ $res != 0 ]; then - grep -F 'Could not solve for environment specs' "$fullbase\\create.log" || exit $res - # run it again but without quiet so it can actually be debugged. - run_mamba -yn "$env" create -c "$channel" \ - "$python" "${main_packages[@]}" 2>&1 \ - | tee "$fullbase\\create.log" - exit ${PIPESTATUS[0]} + [ $res = 0 ] || exit "$res" + + if [ $# = 0 ]; then + common=(--isolated --disable-pip-version-check) + run_pip install "${common[@]}" --no-deps "${pypi_packages[@]}" 2>&1 \ + | tee -a "$fullbase/create.log" + [ ${PIPESTATUS[0]} = 0 ] || exit ${PIPESTATUS[0]} + # using run_python is faster than run_pip since it won't check for compilers. + run_python -m pip "${common[@]}" check \ + | tee -a "$fullbase/create.log" + #[ ${PIPESTATUS[0]} = 0 ] || exit ${PIPESTATUS[0]} fi - common=(--isolated --disable-pip-version-check) - run_pip install "${common[@]}" --no-deps "${pypi_packages[@]}" 2>&1 \ - | tee -a "$fullbase\\create.log" - [ ${PIPESTATUS[0]} = 0 ] || exit ${PIPESTATUS[0]} - # using run_python is faster than run_pip since it won't check for compilers. - run_python -m pip "${common[@]}" check \ - | tee -a "$fullbase\\create.log" - #[ ${PIPESTATUS[0]} = 0 ] || exit ${PIPESTATUS[0]} - - pre="$fullbase\\$env" + pre="$fullbase/$env" run_mamba -n "$env" env export > "$pre.yml" || exit run_mamba -n "$env" list --json | run_python -c \ - 'for p in (m:=__import__)("json").load(m("sys").stdin):print(p["name"]+"=="+p["version"])' \ + 'for(p)in(m:=__import__)("json").load(m("sys").stdin):print(p["name"]+"=="+p["version"])' \ > "$pre.conda.freeze.txt" || exit [ ${PIPESTATUS[0]} = 0 ] || exit ${PIPESTATUS[0]} @@ -347,7 +455,7 @@ if [ "$env" = ERROR ] || [ $creating != 0 ]; then echo '- pip:' | cat "$pre.yml" - > "$pre.with-pip.yml" || exit run_python -c \ - '[print(*" -",b[k]) for a,b in [map(lambda f:{l.strip().lower().replace(*"-_"):l.strip() for l in open(f)},__import__("sys").argv[1:])] for k in b if k not in a]' \ + '[print(*" -",b[k])for(a,b)in[map(lambda f:{l.strip().lower().replace(*"-_"):l.strip()for(l)in open(f)},__import__("sys").argv[1:])]for(k)in(b)if(k)not in a]' \ "$pre.conda.freeze.txt" "$pre.pip.freeze.txt" \ >> "$pre.with-pip.yml" || exit