From 2a34f38932be344e440c5383748b3a3c37b488ba Mon Sep 17 00:00:00 2001 From: Connor Olding Date: Mon, 13 Jun 2022 06:36:34 +0200 Subject: [PATCH] add automamba --- automamba/README.md | 0 automamba/automamba | 359 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 359 insertions(+) create mode 100644 automamba/README.md create mode 100644 automamba/automamba diff --git a/automamba/README.md b/automamba/README.md new file mode 100644 index 0000000..e69de29 diff --git a/automamba/automamba b/automamba/automamba new file mode 100644 index 0000000..b97be1a --- /dev/null +++ b/automamba/automamba @@ -0,0 +1,359 @@ +#!/usr/bin/env bash + +home="${USERPROFILE}" +downloads="$home\\Downloads" +base="$home\\mamba" +channel=main +#channel=conda-forge + +# micromamba stuff; doesn't affect python itself: +installer_version="0.23.3" +installer_sha256="d133dc61615b03fa953b6563c6d0f7362c5319c1ed164a4c1311b7f5e1868776" +installer_archive="micromamba-$installer_version-0.tar.bz2" +installer_remote="https://micromamba.snakepit.net/api/micromamba/win-64/$installer_version" +installer_path="Library/bin/micromamba.exe" + +# remember not to use == when you mean = here: +python='python=3.10' + +main_packages=( + 'pip>=21.2.4' # python package manager +) + +# pip packages should be pinned with pure equality +# because they're not exported by mamba env export. +pypi_packages=( + 'hipsterplot==0.1' # terminal plotting +) + +# end basic settings, begin functions: + +check_installer() { + [ -s "$1" ] || return + printf '%s %s\n' "$installer_sha256" "$1" | sha256sum -c - || return +} + +run_mamba() { + # TODO: infer System32 path by envvars. (it's looking for chcp.exe) + PATH="\ +C:\\Windows\\System32:\ +C:\\Windows\\" \ + "$fullbase\\Library\\bin\\micromamba.exe" --no-rc --no-env -r "$base" "$@" +} + +run_python() { + PATH="\ +$fullbase\\envs\\$env:\ +$fullbase\\envs\\$env\\Library\\mingw-w64\\bin:\ +$fullbase\\envs\\$env\\Library\\bin" \ + "$fullbase\\envs\\$env\\python.exe" "$@" +} + +find_msvc() { + [ -n "$SYSTEMDRIVE" ] || { echo '$SYSTEMDRIVE must be set.' >&2; exit 1; } + + dirs= libs= lib= includes= + local dir= temp= + local profiles="Program Files (x86)" + + cd "$SYSTEMDRIVE" || return + [ -d "$profiles" ] || return + + add_dir() { + dir="$1" + dir="${dir%/}" + dir="$(readlink -f "$dir")" || { printf 'readlink failed: %s\n' "$1" >&2; return 1; } + if [ ! -d "$dir" ] && [ -e "$dir" ]; then + dir="${dir%/*}" + fi + [ -d "$dir" ] || { printf 'not a directory: %s\n' "$dir"; return 1; } + dirs+=":$dir" + } + + windir() { + # TODO: don't use cygpath, somehow. + temp="$(cygpath -w "$1")" || return + [ -d "$temp" ] || return + } + + dir="$profiles/Microsoft Visual Studio" + [ -d "$dir" ] || return + dir="$(find "$dir" -name cl.exe | grep -E '[\/]Hostx64[\/]x64[\/]cl.exe$' | LC_ALL=C sort | tail -n1)" + add_dir "$dir" || return + + windir "${dir%/bin/*}/lib/x64" || return; libs+=";$temp"; lib="$temp" + windir "${dir%/bin/*}/include" || return; includes+=";$temp" + + dir="$profiles/Windows Kits" + [ -d "$dir" ] || return + dir="$(find "$dir" -name rc.exe | grep -E '[\/]x64[\/]rc.exe' | LC_ALL=C sort | tail -n1)" + add_dir "$dir" || return + + # NOTE: this next line relies on add_dir behavior a little. + windir "$dir/ucrt" || return; includes+=";$temp" + + dir="${dir%/x64}" + windir "${dir/bin/Lib}/um/x64" || return; libs+=";$temp" + windir "${dir/bin/Lib}/ucrt/x64" || return; libs+=";$temp" + windir "${dir/bin/Include}/ucrt" || return; includes+=";$temp" + windir "${dir/bin/Include}/shared" || return; includes+=";$temp" + + dirs="${dirs#:}" + libs="${libs#;}" + includes="${includes#;}" + #printf '%s\n' "DIRS: $dirs" >&2 + #printf '%s\n' "LIBS: $libs" >&2 + #printf '%s\n' "INCLUDES: $includes" >&2 + #exit 1 +} + +install_msvc() { + [ -n "$SYSTEMDRIVE" ] || { echo '$SYSTEMDRIVE must be set.' >&2; exit 1; } + [ -n "$COMSPEC" ] || { echo '$COMSPEC must be set.' >&2; exit 1; } + + # find the latest version: curl -I 'https://aka.ms/vs/17/release/vs_buildtools.exe' + local uuid='05734053-383e-4b1a-9950-c7db8a55750d' + local hash='fbfc005ace3e6b4990e9a4be0fa09e7face1af5ee1f61035c64dbc16c407aeda' + local exe='vs_BuildTools.exe' + local installer="$downloads\\$exe" + local url="https://download.visualstudio.microsoft.com/download/pr/$uuid/$hash/$exe" + local profiles="Program Files (x86)" + + [ -s "$installer" ] || curl -L "$url" -o "$installer" || return + printf '%s %s\n' "$hash" "$installer" | sha256sum -c - || return + + cd "$SYSTEMDRIVE" || return + [ -d "$profiles" ] || return + + local bt + bt="$profiles\\Microsoft Visual Studio\\2022\\BuildTools" + + # TODO: use "update" instead of "install" when applicable. + # despite the name, you can use the Windows 11 SDK (or at least this version of it) + # to target as early as Windows 10 v1507. + "$installer" --quiet --wait --norestart --nocache install \ + --add 'Microsoft.VisualStudio.Component.VC.Tools.x86.x64' \ + --add 'Microsoft.VisualStudio.Component.Windows11SDK.22000' + local ret=$? + + # bash has run ERRORLEVEL through a modulus of 256, so some hacks are required. + local code=0 + case $ret in + (0) code=0;; # success (probably) + (58) code=-1073741510;; + (65) code=8001;; + (66) code=1602;; # NOTE: this could also be 8002! + (67) code=8003;; + (68) code=8004;; + (69) code=8005;; + (70) code=8006;; + (82) code=1618;; + (87) code="INVALID ARGUMENTS ($ret)";; + (105) code=1641;; + (139) code=5003;; + (140) code=5004;; + (141) code=5005;; + (143) code=5007;; + (145) code=-1073720687;; + (194) code=3010;; # success; restart required (but not really) + (228) code=740;; + (233) code=1001;; + (235) code=1003;; + (*) code="UNKNOWN ($ret)";; + esac + + printf '%s exited with %%ERRORLEVEL%% %s.\n' "$installer" "$code" >&2 + [ "$code" = 0 ] || [ "$code" = 3010 ] || return 1 +} + +run_with_compiler() { + find_msvc || install_msvc \ + || { printf 'failed to install MSVC and a Windows SDK.\n' >&2; exit 1; } + find_msvc \ + || { printf 'failed to find a valid installation of MSVC.\n' >&2; exit 1; } + + cd "$working_dir" || return + PATH="\ +$fullbase\\envs\\$env:\ +$fullbase\\envs\\$env\\Library\\mingw-w64\\bin:\ +$fullbase\\envs\\$env\\Library\\bin:\ +$dirs" \ + LIB="$libs" \ + LIBPATH="$lib" \ + INCLUDE="$includes" \ + python.exe "$@" +} + +run_pip() { + run_with_compiler -m pip --disable-pip-version-check "$@" +} + +find_last_env() { + local arg="$1" e= oe= + env="$(date -u '+%Y%m%d')" || exit + n=1 + while [ -d "$fullbase\\envs\\$env-$n" ]; do + let n++ + done + new_env="$env-$n" + let n-- + if [ $n = 0 ]; then + env=_ + if [ -d "$fullbase\\envs" ]; then + for e in "$fullbase\\envs/"*; do # must use a forward slash for some reason + oe="${e##*/}" + #printf 'considering %s... compared to %s\n' "$oe" "$env" >&2 + if [ -d "$e" ] && [ "${oe#20}" != "$oe" ] && [[ "$oe" > "$env" ]]; then + #printf '%s is more recent!\n' "$oe" + env="$oe" + fi + done + fi + if [ "$env" = _ ]; then + env=ERROR + if [ "$arg" != new ]; then + echo 'failed to find most recent environment' >&2 + exit 1 + fi + fi + else + env="$env-$n" + fi +} + +# end of functions, begin main: + +working_dir="$PWD" + +[ -n "$USERPROFILE" ] || { echo '$USERPROFILE must be set.' >&2; exit 1; } +[ -d "$working_dir" ] || { echo 'wat?' >&2; exit 1; } + +[ -d "$base" ] || mkdir "$base" || exit +fullbase="$(cygpath -u "$base")" || exit +fullbase="$(readlink -f "$fullbase")" || exit +fullbase="$(cygpath -w "$fullbase")" || exit + +version="unknown" +if [ -s "$fullbase\\$installer_path" ]; then + # TODO: do something meaningful if the execution itself fails. + version="$(run_mamba --version)" +fi + +if [ "$version" != "$installer_version" ]; then + archive="$downloads\\$installer_archive" + if check_installer "$archive"; then + : + else + # slightly less safe: + # curl -LOJ "$installer_remote" + curl -L "$installer_remote" -o "$archive" || exit + check_installer "$archive" || exit + fi + archive="$(cygpath -u "$archive")" || exit + cd "$fullbase" || exit + tar jxf "$archive" "$installer_path" || exit + cd "$working_dir" || exit +fi + +[ "$1" = --create ] && creating=1 || creating=0 +[ "$1" = --mamba ] && dancing=1 || dancing=0 +[ "$1" = --pip ] && compiling=1 || compiling=0 +[ "$1" = --wheel ] && wheeling=1 || wheeling=0 + +find_last_env new || exit + +if [ $dancing = 1 ]; then + [ $dancing = 0 ] || shift || exit + + no_default=0 + for arg in "$@"; do + [ "$arg" != --name ] || no_default=1 + # TODO: is --name=stuff valid? detect that too. + if [ "${arg#--}" = "$arg" ]; then # ignore other long arguments + # detect -n, -asdfn shortargs. startswith -, endswith n. + if [ "${arg#-}" != "$arg" ] && [ "${arg%n}" != "${arg}" ]; then + no_default=1 + fi + fi + done + if [ $no_default = 0 ]; then + run_mamba -n "$env" "$@" || exit + else + run_mamba "$@" || exit + fi + + [ $dancing = 0 ] || exit 0 +fi + +if [ $compiling = 1 ]; then + [ $compiling = 0 ] || shift || exit + + [ "$env" != ERROR ] || { printf 'failed to find most recent environment\n' >&2; exit 1; } + run_pip "$@" || exit + + [ $compiling = 0 ] || exit 0 +fi + +if [ $wheeling = 1 ]; then + [ $wheeling = 0 ] || shift || exit + + [ "$env" != ERROR ] || { printf 'failed to find most recent environment\n' >&2; exit 1; } + run_with_compiler -u setup.py bdist_wheel || exit + + [ $wheeling = 0 ] || exit 0 +fi + +if [ "$env" = ERROR ] || [ $creating != 0 ]; then + [ $creating = 0 ] || shift || exit + env="$new_env" + + if [ $creating = 0 ]; then + printf 'WARNING: automatically creating a new environment: %s' "$env" >&2 + fi + + run_mamba -yqn "$env" create -c "$channel" \ + "$python" "${main_packages[@]}" 2>&1 \ + | tee "$fullbase\\create.log" + res=${PIPESTATUS[0]} + if [ $res != 0 ]; then + grep -F 'Could not solve for environment specs' "$fullbase\\create.log" || exit $res + # run it again but without quiet so it can actually be debugged. + run_mamba -yn "$env" create -c "$channel" \ + "$python" "${main_packages[@]}" 2>&1 \ + | tee "$fullbase\\create.log" + exit ${PIPESTATUS[0]} + fi + + common=(--isolated --disable-pip-version-check) + run_pip install "${common[@]}" --no-deps "${pypi_packages[@]}" 2>&1 \ + | tee -a "$fullbase\\create.log" + [ ${PIPESTATUS[0]} = 0 ] || exit ${PIPESTATUS[0]} + # using run_python is faster than run_pip since it won't check for compilers. + run_python -m pip "${common[@]}" check \ + | tee -a "$fullbase\\create.log" + #[ ${PIPESTATUS[0]} = 0 ] || exit ${PIPESTATUS[0]} + + pre="$fullbase\\$env" + + run_mamba -n "$env" env export > "$pre.yml" || exit + + run_mamba -n "$env" list --json | run_python -c \ + 'for p in (m:=__import__)("json").load(m("sys").stdin):print(p["name"]+"=="+p["version"])' \ + > "$pre.conda.freeze.txt" || exit + [ ${PIPESTATUS[0]} = 0 ] || exit ${PIPESTATUS[0]} + + run_python -m pip "${common[@]}" list --pre --format=freeze \ + > "$pre.pip.freeze.txt" || exit + + echo '- pip:' | cat "$pre.yml" - > "$pre.with-pip.yml" || exit + run_python -c \ + '[print(*" -",b[k]) for a,b in [map(lambda f:{l.strip().lower().replace(*"-_"):l.strip() for l in open(f)},__import__("sys").argv[1:])] for k in b if k not in a]' \ + "$pre.conda.freeze.txt" "$pre.pip.freeze.txt" \ + >> "$pre.with-pip.yml" || exit + + rm "$pre.conda.freeze.txt" "$pre.pip.freeze.txt" || exit + + [ $creating = 0 ] || exit 0 +fi + +run_python "$@"