diff --git a/bin/cherry-py-repo-from-template b/bin/cherry-py-repo-from-template index 7112d1b..fe0fbbf 100755 --- a/bin/cherry-py-repo-from-template +++ b/bin/cherry-py-repo-from-template @@ -1,33 +1,33 @@ -#!/bin/bash +#!/usr/bin/env bash TEMPLATE_DIR="swh-py-template" usage () { echo "Usage: cherry-py-repo-from-template GIT_REVISION [REPO_NAME]" echo echo "Apply the GIT_REVISION change of the swh-py-template repository to either" echo "REPO_NAME, or all Python repositories" exit 1 } if [ -z "$1" ] ; then usage fi gitrev="$1" shift 1 apply_to_repo () { repo="$1" echo "* Applying to repo $1..." (cd "$TEMPLATE_DIR" && git show "$gitrev") | (cd "$repo" ; git apply) } if [ -n "$1" ] ; then repo="$1" apply_to_repo "$repo" else for repo in `bin/ls-py-modules` ; do apply_to_repo "$repo" done echo "All done." fi diff --git a/bin/debpkg-bootstrap-branches b/bin/debpkg-bootstrap-branches index 7c4e10c..37c3064 100755 --- a/bin/debpkg-bootstrap-branches +++ b/bin/debpkg-bootstrap-branches @@ -1,109 +1,109 @@ -#!/bin/bash +#!/usr/bin/env bash set -e set -x set -o pipefail package=$(basename $(pwd)) module=${package//-/.} if [ $# -ne 0 ]; then last_debian_rev=$1 else last_debian_rev=master fi git branch -D pristine-tar debian/upstream debian/unstable-swh debian/stretch-swh || true for tag in `git tag -l debian/*`; do git tag -d $tag done for tag in `git tag -l --sort=v:refname v\*`; do ver=${tag/v/} if [ -f ../packages/${package}_${ver}.orig.tar.gz ]; then continue fi git checkout $tag if [ -d swh ]; then git clean -dfx swh fi if [ -f yarn.lock ]; then (yarn install --frozen-lockfile && yarn build) || true fi find . -maxdepth 1 -type d -name '*.egg-info' -exec rm -r '{}' \+ python3 setup.py egg_info pname=$(awk '/^Name:/{print $2}' *.egg-info/PKG-INFO) pver=$(awk '/^Version:/{print $2}' *.egg-info/PKG-INFO) python3 setup.py sdist -d ../packages mv ../packages/${pname}-${pver}.tar.gz ../packages/${package}_${ver}.orig.tar.gz done upstream_tag=v0.0.1 ver=0.0.1 author_name=$(git tag -l --format="%(if)%(*objecttype)%(then)%(taggername)%(else)%(authorname)%(end)" "${upstream_tag}") author_email=$(git tag -l --format="%(if)%(*objecttype)%(then)%(taggeremail)%(else)%(authoremail)%(end)" "${upstream_tag}") # Strip <> author_email=${author_email:1:-1} author_date=$(git tag -l --format="%(if)%(*objecttype)%(then)%(taggerdate:iso)%(else)%(authordate:iso)%(end)" "${upstream_tag}") export DEBEMAIL="${author_email}" export DEBFULLNAME="${author_name}" export GIT_AUTHOR_NAME="${author_name}" export GIT_AUTHOR_EMAIL="${author_email}" export GIT_AUTHOR_DATE="${author_date}" export GIT_COMMITTER_NAME="${author_name}" export GIT_COMMITTER_EMAIL="${author_email}" export GIT_COMMITTER_DATE="${author_date}" git checkout $upstream_tag git clean -dfx git checkout -b debian/upstream git ls-tree --name-only HEAD | xargs rm -r tar -x --strip-components 1 -f ../packages/${package}_0.0.1.orig.tar.gz git add . git commit --no-verify -m "Import upstream version 0.0.1" git tag debian/upstream/0.0.1 git checkout --orphan pristine-tar pristine-tar commit ../packages/${package}_0.0.1.orig.tar.gz git checkout debian/upstream git checkout -b debian/unstable-swh git checkout ${last_debian_rev} -- debian cat > debian/gbp.conf << EOF [DEFAULT] upstream-branch=debian/upstream upstream-tag=debian/upstream/%(version)s upstream-vcs-tag=v%(version)s debian-branch=debian/unstable-swh pristine-tar=True EOF rm debian/changelog faketime "${author_date}" dch --create --package ${package} -v 0.0.1-1~swh1 'Initial release' git tag -l --format="%(contents:subject)%(if)%(contents:body)%(then)%0a%(contents:body)%(end)" "${upstream_tag}" | sed -E -e '/^$/d' -e 's/^ *(- *)?//' | while read line; do faketime "${author_date}" dch "${line}" done faketime "${author_date}" dch -D unstable-swh --force-distribution '' git add debian git commit --no-verify -m "Updated debian directory for version 0.0.1" git checkout -b debian/stretch-swh faketime "${author_date}" dch --bpo -D stretch-swh --force-distribution '' git add debian/changelog sed -i s/unstable/stretch/ debian/gbp.conf git add debian/gbp.conf git commit --no-verify -m "Updated debian stretch backport directory for version 0.0.1" for tag in `git tag -l --sort=v:refname v\* | tail -n +2`; do version=${tag/v/} echo $tag: $version ../bin/debpkg-bump-version $version ../packages/${package}_${version}.orig.tar.gz done diff --git a/bin/debpkg-bump-version b/bin/debpkg-bump-version index 624b223..978d51e 100755 --- a/bin/debpkg-bump-version +++ b/bin/debpkg-bump-version @@ -1,75 +1,75 @@ -#!/bin/bash +#!/usr/bin/env bash set -e if [ $# -lt 2 -o $# -gt 3 ]; then echo "usage: $0 []" exit 2 fi newver="$1" shift sdist="$1" shift if [ $# -eq 1 ]; then upstream_tag=$1 shift else upstream_tag="v${newver}" fi debian_tag="debian/upstream/${newver}" echo 'Updating tags...' git fetch --tags if git show-ref --quiet "refs/tags/${debian_tag}"; then echo "Tag ${debian_tag} already exists!" exit 0 fi if ! git show-ref --quiet "refs/tags/${upstream_tag}"; then echo "No tag ${upstream_tag} exists, aborting!" exit 3 fi if ! git diff-index --quiet HEAD --; then echo "Git repository not clean, aborting!" git status exit 4 fi author_name=$(git tag -l --format="%(if)%(*objecttype)%(then)%(taggername)%(else)%(authorname)%(end)" "${upstream_tag}") author_email=$(git tag -l --format="%(if)%(*objecttype)%(then)%(taggeremail)%(else)%(authoremail)%(end)" "${upstream_tag}") # Strip <> author_email=${author_email:1:-1} author_date=$(git tag -l --format="%(if)%(*objecttype)%(then)%(taggerdate:iso)%(else)%(authordate:iso)%(end)" "${upstream_tag}") export DEBEMAIL="${author_email}" export DEBFULLNAME="${author_name}" export GIT_AUTHOR_NAME="${author_name}" export GIT_AUTHOR_EMAIL="${author_email}" export GIT_AUTHOR_DATE="${author_date}" export GIT_COMMITTER_NAME="${author_name}" export GIT_COMMITTER_EMAIL="${author_email}" export GIT_COMMITTER_DATE="${author_date}" git checkout debian/unstable-swh gbp import-orig -u "${newver}" "${sdist}" faketime "${author_date}" dch -v "${newver}-1~swh1" '' git tag -l --format="%(contents:subject)%(if)%(contents:body)%(then)%0a%(contents:body)%(end)" "${upstream_tag}" | sed -E -e '/^$/d' -e 's/^ *(- *)?//' | while read line; do faketime "${author_date}" dch "${line}" done faketime "${author_date}" dch -D unstable-swh --force-distribution '' git add debian/changelog git commit --no-verify -m "Updated debian directory for version ${newver}" git checkout debian/stretch-swh git merge debian/unstable-swh --no-commit --no-edit || true git checkout debian/unstable-swh -- debian/changelog git add debian/changelog git commit --no-verify --no-edit faketime "${author_date}" dch --bpo -D stretch-swh --force-distribution '' git add debian/changelog git commit --no-verify -m "Updated debian stretch backport directory for version ${newver}" diff --git a/bin/init-py-repo b/bin/init-py-repo index 342088a..d5467af 100755 --- a/bin/init-py-repo +++ b/bin/init-py-repo @@ -1,27 +1,27 @@ -#!/bin/bash +#!/usr/bin/env bash TEMPLATE_DIR="swh-py-template" usage () { echo "Usage: init-py-repo REPO_NAME" echo echo "Note: the repo should have already been created on the forge." echo "and cloned to the local directory REPO_NAME." exit 1 } if [ -z "$1" -o ! -d "$1" ] ; then usage fi repo_dir="$1" shift 1 if ! [ -d "$TEMPLATE_DIR" ] ; then echo "can't find template directory: $TEMPLATE_DIR. Abort" exit 2 fi rsync -rv --exclude='**/.git/**' "$TEMPLATE_DIR/" "$repo_dir/" cd "$repo_dir" git add . git commit -m "import template from $TEMPLATE_DIR (init-py-repo)" diff --git a/bin/ls-all-repos b/bin/ls-all-repos index 12cc082..71dd7f6 100755 --- a/bin/ls-all-repos +++ b/bin/ls-all-repos @@ -1,42 +1,42 @@ -#!/bin/bash +#!/usr/bin/env bash cd "$( dirname "$0" )/.." abspath=no if [ "$1" == "-a" -o "$1" == "--absolute" ] ; then abspath=yes fi warn () { echo "W: $@" 1>&2 } # list known repositories mr -t list \ | grep '^mr list:' | grep -v '^mr list: finished' \ | awk '{print $3}' \ | while read repo_abspath ; do if [ "$abspath" == "no" ] ; then basename $repo_abspath else echo $repo_abspath fi done # list obsolete repositories find . -mindepth 1 -maxdepth 1 -type d -not -path '*/\.*' | while read dir ; do dir=${dir#./} # strip "./" prefix if [ "$dir" == ".git" \ -o "$dir" == "bin" \ -o "$dir" == "doc" \ -o "$dir" == "docker" \ -o "$dir" == "packages" \ ] ; then continue fi if ! grep -q "^\[${dir}\]" .mrconfig ; then warn "unknown repository '$dir', you might want to remove it" fi done diff --git a/bin/ls-py-modules b/bin/ls-py-modules index 661985f..604afec 100755 --- a/bin/ls-py-modules +++ b/bin/ls-py-modules @@ -1,9 +1,9 @@ -#!/bin/bash +#!/usr/bin/env bash LS_REPOS="bin/ls-all-repos" if ! [ -x "$LS_REPOS" ] ; then echo "ls-py-modules should be run from the root of swh-environment. Bye." exit 2 fi "$LS_REPOS" "$@" | egrep -v -- "(-(template|testdata)$|snippets|docker)" diff --git a/bin/make-package b/bin/make-package index 4814c2a..69f6dbb 100755 --- a/bin/make-package +++ b/bin/make-package @@ -1,130 +1,130 @@ -#!/bin/bash +#!/usr/bin/env bash usage() { echo "Usage: $0 [-b|--build] [-u|--upload] [-d|--distribution ] SWH_PACKAGE" echo "E.g.: make-package -b swh-core" exit 1 } # command line parsing build="no" upload="no" package="" distribution="all" while (( "$#" )); do case "$1" in -b|--build) build="yes" ;; -u|--upload) upload="yes" ;; -d|--distribution) shift; distribution=$1;; *) package="$1";; esac shift done if [ "$build,$upload" = "no,no" -o -z "$package" ] ; then usage fi if [ $distribution != 'unstable' -a $distribution != 'stable' -a $distribution != 'all' ]; then usage fi set -e CURDIR=$(readlink -f "$package") PACKAGEDIR=$(readlink -f "packages") BASENAME="$(basename "$CURDIR")" MODULE="${BASENAME//-/.}" REPOSITORY=https://debian.softwareheritage.org/ DESTINATION=pergamon.internal.softwareheritage.org DESTDIR=/srv/softwareheritage/repository TEMP=$(mktemp -d) trap "{ rm -rf $TEMP; }" EXIT cd "$CURDIR" VERSION=$(python3 -c 'import vcversioner; print(vcversioner.find_version().version)') SID_VERSION=${VERSION}-1 SID_CHANGES_FILE=${BASENAME}_${SID_VERSION}_amd64.changes SID_LOGFILE=${BASENAME}_${SID_VERSION}_amd64.build BPO_VERSION=${SID_VERSION}~bpo9~swh+1 BPO_CHANGES_FILE=${BASENAME}_${BPO_VERSION}_amd64.changes BPO_LOGFILE=${BASENAME}_${BPO_VERSION}_amd64.build SBUILD="sbuild -As --force-orig-source --build-dep-resolver=aptitude --build-failed-commands %SBUILD_SHELL --no-run-lintian" if [ "$build" = "yes" ] ; then # Generate source tarball and put it in the right place python3 setup.py sdist -d $TEMP mv $TEMP/$MODULE-$VERSION.tar.gz $TEMP/${BASENAME}_${VERSION}.orig.tar.gz # Extract source tarball and overlay Debian packaging cd $TEMP tar xf ${BASENAME}_${VERSION}.orig.tar.gz mv $MODULE-$VERSION $BASENAME-$VERSION cd $BASENAME-$VERSION cp -r $CURDIR/debian . if [ "$distribution" = "all" -o "$distribution" = "unstable" ]; then # Generate changelog for unstable dch -v "${SID_VERSION}" "Deploy ${VERSION}" dch --force-distribution --distribution unstable-swh -r "" # Build unstable package with original source $SBUILD \ --extra-repository="deb [trusted=yes] ${REPOSITORY} unstable main" \ --extra-repository="deb http://incoming.debian.org/debian-buildd/ buildd-unstable main" # Copy package to staging directory dcmd cp ../${SID_CHANGES_FILE} ${PACKAGEDIR} cp -L ../${SID_LOGFILE} ${PACKAGEDIR} fi if [ "$distribution" = "all" -o "$distribution" = "stable" ]; then # Generate changelog for backports dch -bv "${BPO_VERSION}" "Rebuild for stretch-backports-swh" dch -r --distribution stretch-backports-swh --force-distribution "" # Build backport package $SBUILD \ --extra-repository="deb [trusted=yes] ${REPOSITORY} stretch-swh main" \ --extra-repository="deb http://deb.debian.org/debian stretch-backports main" \ --extra-repository="deb https://download.ceph.com/debian-luminous/ stretch main" --extra-repository-key=${PACKAGEDIR}/keys/ceph.asc # Copy package to staging directory dcmd cp ../${BPO_CHANGES_FILE} ${PACKAGEDIR} cp -L ../${BPO_LOGFILE} ${PACKAGEDIR} fi fi cd "$CURDIR" if [ "$upload" = "yes" ] ; then changefiles=() if [[ "${VERSION}" == *dev* || "${VERSION}" == *post* ]]; then echo "Uploading a dev version is not allowed! Please tag and rebuild." exit 2 fi if [ "$distribution" = "all" -o "$distribution" = "unstable" ]; then changefiles+=(${PACKAGEDIR}/${SID_CHANGES_FILE}) fi if [ "$distribution" = "all" -o "$distribution" = "stable" ]; then changefiles+=(${PACKAGEDIR}/${BPO_CHANGES_FILE}) fi # Sign and send packages for changefile in "${changefiles[@]}"; do debsign ${changefile} dcmd scp ${changefile} ${DESTINATION}:${DESTDIR}/incoming ssh ${DESTINATION} "umask 002; reprepro -vb ${DESTDIR} processincoming incoming" done git push --tags fi diff --git a/bin/pip-ls-deps b/bin/pip-ls-deps index 899e119..edbb597 100755 --- a/bin/pip-ls-deps +++ b/bin/pip-ls-deps @@ -1,27 +1,27 @@ -#!/bin/bash +#!/usr/bin/env bash LS_MODS="bin/ls-py-modules" if ! [ -x "$LS_MODS" ] ; then echo "pip-ls-deps should be run from the root of swh-environment. Bye." exit 2 fi for pymod in $(${LS_MODS}) ; do reqs="${pymod}/requirements.txt" if [ -f "$reqs" ] ; then cat "$reqs" fi done \ | egrep -v '^#|^[[:blank:]]*$' \ | tr '[:upper:]_' '[:lower:]-' \ | sort -ru \ | awk '!seen[$1]++' \ | tac # WARNING: CRUDE HACK in the pipeline above. 'pip3 install' will fail if the # same package name appears multiple times, e.g., as both "django" and "django # >= 1.10.7" rather than trying to determine (when possible) a comprehensive # constraint. So we deduplicate here, by first sorting alphabetically which # will (generally...) put higher dependency constraints last and then # deduplicate using the sole package name as key. In the example above this # should only keep "django >= 1.10.7" and get rid of "django". diff --git a/bin/pip-swh-packages b/bin/pip-swh-packages index 0e73734..73506d7 100755 --- a/bin/pip-swh-packages +++ b/bin/pip-swh-packages @@ -1,19 +1,19 @@ -#!/bin/bash +#!/usr/bin/env bash cd "$( dirname $0 )/.." suffix="" if [ "$1" == "--with-testing" ]; then suffix="[testing]" fi ./bin/py-depgraph | grep -- '->' | sed -e 's/[";]//g' -e 's/->//g' | tsort | tac | grep '^swh\.' | sed 's:\.:-:g' | sed 's:^:-e :' | sed "s/$/$suffix/" | sed -e 's/\]\[/,/g' diff --git a/bin/py-depgraph b/bin/py-depgraph index 7429e4a..e852dad 100755 --- a/bin/py-depgraph +++ b/bin/py-depgraph @@ -1,62 +1,62 @@ -#!/bin/bash +#!/usr/bin/env bash # generate dependency graph (in DOT(1) format) for all known python modules # # include by default both internal and external dependencies, but can asked to # include either or none of them (see --help) internal_modules=1 external_modules=1 while [ -n "$1" ] ; do if [ "$1" = "--no-internal" ] ; then internal_modules=0 elif [ "$1" = "--no-external" ] ; then external_modules=0 elif [ "$1" = "--help" -o "$1" = "-h" ] ; then echo "Usage: bin/py-depgraph [--no-internal] [--no-external] > FILE.dot" exit 1 fi shift 1 done pyrepos=$(bin/ls-py-modules) # available python repositories (with '-') declare -A pymods # available python modules (with '.') for repo in $pyrepos ; do pymod=${repo//-/.} pymods[$pymod]=1 done echo "digraph swh_py_deps {" for pymod in ${!pymods[@]} ; do echo -e "\t\"$pymod\" ;" done getdeps() { grep -E -v '(^#|^[[:space:]]*$)' "$1" | cut -f 1 -d ' ' | tr 'A-Z' 'a-z' | sed -E 's/\[\w+\]//' } for repo in $pyrepos ; do pymod=${repo//-/.} reqs_int="${repo}/requirements-swh.txt" reqs_ext="${repo}/requirements.txt" if [ "$internal_modules" -eq 1 -a -f "$reqs_int" ]; then for dep in $( getdeps "$reqs_int" ) ; do echo -e "\t\"${pymod}\" -> \"${dep}\" ;" done fi if [ "$external_modules" -eq 1 -a -f "$reqs_ext" ]; then for dep in $( getdeps "$reqs_ext" ) ; do echo -e "\t\"${dep}\" [style=dashed] ;" echo -e "\t\"${pymod}\" -> \"${dep}\" ;" done fi done | sort -u echo "}" diff --git a/bin/pypi-check-versions b/bin/pypi-check-versions index 9efe671..cf839c6 100755 --- a/bin/pypi-check-versions +++ b/bin/pypi-check-versions @@ -1,19 +1,19 @@ -#!/bin/bash +#!/usr/bin/env bash mr -m run bash -c ' if [[ $(basename $PWD) =~ ^(swh-docs|swh-py-template)$ ]]; then echo SKIP $(basename $PWD) else if [ -f setup.py ] ; then PKG=$(python3 setup.py --name) if [ -n "$PKG" ]; then PYPI=v$(http https://pypi.org/pypi/$PKG/json | jq -r .info.version) LASTTAG=$(git tag --sort v:refname | tail -n1) if [ v"$LASTTAG" == v"$PYPI" ]; then echo "OK $PKG $LASTTAG" else (>&2 echo "$PKG $LASTTAG (git) != $PYPI (pypi)") fi fi fi fi ' | grep -v 'mr run' diff --git a/bin/update b/bin/update index 269c5b1..fce896e 100755 --- a/bin/update +++ b/bin/update @@ -1,17 +1,17 @@ -#!/bin/bash +#!/usr/bin/env bash mrconf=$(readlink -f .mrconfig) mrtrust="${HOME}/.mrtrust" if ! grep -q "$mrconf" "$mrtrust" &> /dev/null ; then echo "I: mr repository not found in ${mrtrust}, adding it." echo "$mrconf" >> "$mrtrust" fi if ! which pre-commit &> /dev/null ; then echo "The pre-commit command is required, please install it before" echo "running this command. See README.md for more information." exit 1 fi git pull mr -j 4 update