Initial import

get-aquila-modules.sh

@@ -0,0 +1,616 @@
+#!/bin/sh
+#+
+#   ARES/HADES/BORG Package -- ./get-aquila-modules.sh
+#   Copyright (C) 2014-2020 Guilhem Lavaux <guilhem.lavaux@iap.fr>
+#   Copyright (C) 2009-2020 Jens Jasche <jens.jasche@fysik.su.se>
+#
+#   Additional contributions from:
+#      Guilhem Lavaux <guilhem.lavaux@iap.fr> (2023)
+#   
+#+
+
+C_DEFAULT=$(printf "\033[0m")
+C_WHITE=$(printf "\033[1m")
+C_GREEN=$(printf "\033[92m")
+C_RED=$(printf "\033[91m")
+C_BG_RED=$(printf "\033[41m")
+C_BG_GREEN=$(printf "\033[42m")
+
+errormsg() {
+  msg=$1
+  printf "${C_BG_RED}${msg}${C_DEFAULT}\n"
+}
+
+abort() {
+  errormsg "$1"
+  exit 1
+}
+
+trap "printf \"${C_DEFAULT}\"\n" SIGTERM EXIT
+
+print_help()
+{
+  cat<<EOF
+This is the get-aquila-module helper script. It clones and updates the modules
+common to the collaboration. By default it uses SSH protocol. We advise using
+a SSH key to avoid being asked too much about your password.
+
+--hooks            Setup git hooks
+--clone            Clone missing modules
+--https USER       Use HTTPS protocol instead of SSH 
+--pull             Pull all the modules
+--push             Push all modules to origin (including root)
+--help             This information message
+--send-pack H D R  Send packs of ref R to remote ssh server (host H, directory D)
+--local-merge      Use local GIT database to merge the reference to checked 
+                   out branches
+--update-copyright Update copyright notices
+--update-indent    Update indentation
+--status           Show git status
+--report           Create a git version report for reproductibility
+--branch-set       Setup branches of default modules
+                   directory D (including modules)
+--purge            Purge the content of modules (DANGEROUS)
+
+
+Developer tools:
+--tags             Tags all submodule with current version in ARES
+
+Experimental / not working:
+--work-tree B D    Create a complete worktree of branch B in 
+EOF
+}
+
+# ===========================================
+# get current branch in git repo
+function parse_git_branch() {
+     BRANCH=`git branch 2> /dev/null | sed -e '/^[^*]/d' -e 's/* \(.*\)/\1/'`
+     if [ ! "${BRANCH}" == "" ]
+     then
+          STAT=`parse_git_dirty`
+          echo "(${BRANCH}${STAT})"
+     else
+          echo ""
+     fi
+}
+
+# ===========================================
+# get current status of git repo
+function parse_git_dirty {
+     status=`git status 2>&1 | tee`
+     dirty=`printf "${status}" 2> /dev/null | grep "modified:" &> /dev/null; echo "$?"`
+     untracked=`printf "${status}" 2> /dev/null | grep "Untracked files" &> /dev/null; echo "$?"`
+     ahead=`printf "${status}" 2> /dev/null | grep "Your branch is ahead of" &> /dev/null; echo "$?"`
+     newfile=`printf "${status}" 2> /dev/null | grep "new file:" &> /dev/null; echo "$?"`
+     renamed=`printf "${status}" 2> /dev/null | grep "renamed:" &> /dev/null; echo "$?"`
+     deleted=`printf "${status}" 2> /dev/null | grep "deleted:" &> /dev/null; echo "$?"`
+     bits=''
+     if [ "${renamed}" == "0" ]; then
+          bits=">${bits}"
+     fi
+     if [ "${ahead}" == "0" ]; then
+          bits="*${bits}"
+     fi
+     if [ "${newfile}" == "0" ]; then
+          bits="+${bits}"
+     fi
+     if [ "${untracked}" == "0" ]; then
+          bits="?${bits}"
+     fi
+     if [ "${deleted}" == "0" ]; then
+          bits="x${bits}"
+     fi
+     if [ "${dirty}" == "0" ]; then
+          bits="!${bits}"
+     fi
+     if [ ! "${bits}" == "" ]; then
+          echo " ${bits}"
+     else
+          echo ""
+     fi
+}
+
+
+# ===========================================
+# Check whether current git is dirty
+check_dirty()
+{
+  if test $force != 0; then
+    return
+  fi
+
+  d=$1
+  r=$(
+    cd $d;
+    git status --porcelain | grep '^ M'
+  )
+  if test x"$r" != x""; then
+    echo "Module ${d} is not clean."
+    exit 1
+  fi
+}
+
+indent_out()
+{
+  echo "${C_RED} | "
+  echo " ->"
+  sed 's/\r/\n/' | sed "s/^/${C_RED}   |${C_DEFAULT}    /"
+}
+# ===========================================
+# Send notice to user
+#
+
+warning_msg() {
+if [ "x$operation" != xreport ]; then
+  echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+  echo "This script can be run only by Aquila members."
+  echo "if your bitbucket login is not accredited the next operations will fail."
+  echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+fi
+}
+
+if ! command -v git 2>&1 > /dev/null; then
+  echo "Git is required."
+  exit 1
+fi
+
+# ===========================================
+# Argument parsing
+#
+operation=''
+prefix=git@bitbucket.org:
+force=0
+be_quiet=0
+while test $# -gt 0; do
+  key="$1"
+  case $key in
+  --clone)
+     operation="clone"
+     ;;
+  --hooks)
+     operation="hooks"
+     ;;
+  --https)
+     user="$2"
+     prefix="https://${user}@bitbucket.org/"
+     shift
+     ;;
+  --pull)
+     operation="pull"
+     ;;
+  --local-merge)
+     operation="local_merge"
+     [ -z "$2" ] && abort "No source provided"
+     source_ref="$2"
+     shift 1
+     ;;
+  --send-pack)
+     operation="send"
+     [ -z "$2" ] && abort "No host provided"
+     [ -z "$3" ] && abort "No directory provided"
+     [ -z "$4" ] && abort "No git ref provided"
+     target_host="$2"
+     target_directory="$3"
+     target_ref="$4"
+     shift 3
+     ;;
+  --push)
+     operation="push"
+     ;;
+  --purge)
+     operation="purge"
+     ;;
+  --update-copyright)
+     operation="update_copyright"
+     ;;
+  --update-indent)
+     operation="update_indent"
+     ;;
+  --force)
+     force=1
+     ;;
+  --status)
+     operation="status"
+     ;;
+  --report)
+     operation="report"
+     ;;
+  --branch-set)
+     operation="branch_set"
+     ;;
+  --work-tree)
+     operation="worktree"
+     [ -z "$2" ] && abort "No branch provided"
+     [ -z "$3" ] && abort "No directory provided"
+     branch_work=$2
+     branch_dir=$3
+     shift 2
+     ;;
+  --tags)
+     operation="tags"
+     ;;
+  --push-tags)
+     operation="push_tags"
+     ;;
+  -q)
+     be_quiet=1
+     ;;
+  -h|--h|--he|--hel|--help)
+     print_help
+     exit 0
+     ;;
+  esac
+  shift
+done
+
+if [ "$be_quiet" == "0" ]; then
+  warning_msg
+fi
+
+# ===========================================
+
+if test "x${operation}" = "x"; then
+  errormsg "The operation to do is missing."
+  print_help
+  exit 1
+fi
+
+export prefix
+
+# ===========================================
+#
+# Read the information about Aquila modules
+read_submodules() {
+  local submod
+  submod=""
+  while read; do
+    if test "x${REPLY}" = "x"; then 
+      continue
+    fi
+    set -- $REPLY;
+    submod="${submod} $1,$2,$3"
+  done < $1 
+
+  echo ${submod}
+}
+
+close_master() {
+  trap - SIGINT
+  ssh -O exit -o ${cpath} ${target_host}
+  exit 1
+}
+
+get_current_branch() {
+  if git branch --show-current 2> /dev/null; then
+    return 0
+  else
+    git branch 2> /dev/null | grep -E '^\*' | awk '{print $2; }'
+    return $?
+  fi
+}
+
+setup_hooks() {
+  top=$1
+  HOOK_NAMES="applypatch-msg pre-applypatch post-applypatch pre-commit prepare-commit-msg commit-msg post-commit pre-rebase post-checkout post-merge pre-receive update post-receive post-update pre-auto-gc"
+  # assuming the script is in a bin directory, one level into the repo
+  HOOK_DIR=$(git rev-parse --show-toplevel)/.git/hooks
+
+  for hook in $HOOK_NAMES; do
+      # If the hook already exists, is executable, and is not a symlink
+      if [ ! -h $HOOK_DIR/$hook -a -x $HOOK_DIR/$hook ]; then
+          mv $HOOK_DIR/$hook $HOOK_DIR/$hook.local
+      fi
+      # create the symlink, overwriting the file if it exists
+      # probably the only way this would happen is if you're using an old version of git
+      # -- back when the sample hooks were not executable, instead of being named ____.sample
+      ln -s -f ${top}/build_tools/hooks/hooks-wrapper $HOOK_DIR/$hook
+  done
+  echo ${top} > ${HOOK_DIR}/ares_top.cfg
+}
+
+check_hooks() {
+  printf "Checking whether git hooks are there... "
+  HOOK_DIR=$(git rev-parse --show-toplevel)/.git/hooks
+  if [ ! -e ${HOOK_DIR}/ares_top.cfg ]; then
+    echo "It looks not. Installing..."
+    $SHELL $0 --hooks
+  else
+    echo "It looks so. Skipping."
+  fi
+}
+   
+submodules=$(read_submodules .aquila-modules)
+# ===========================================
+
+# ===========================================
+# Execute the requested operation
+#
+case $operation in
+tags)
+   ares_version=$(git describe)
+   release_msg="Release ${ares_version}"
+   for i in $submodules; do
+      IFS=","
+      set -- $i;
+      sub_git=$1
+      sub=$2
+      branch=$3
+      echo "${C_WHITE}Tagging ${sub_git}${C_DEFAULT} into extra/${sub} with ${ares_version} (${release_msg})"
+      (cd extra/$sub; git tag -f -a -m "${release_msg}" ${ares_version}) | indent_out 
+   done
+   ;;
+local_merge)
+   echo "${C_GREEN}Merging from ${source_ref}${C_DEFAULT}"
+   current_branch=$(get_current_branch)
+   echo "Current branch is ${current_branch}"
+   git merge ${source_ref}/${current_branch}
+   for i in $submodules; do
+     IFS=","
+     set -- $i;
+     sub_git=$1
+     sub=$2
+     branch=$3
+     echo "${C_GREEN}Merging $sub with ${source_ref}/$branch...${C_DEFAULT}"
+     ([ -e extra/$sub ] && (cd extra/$sub; git merge "$source_ref/$branch" ) ) || exit 1
+   done
+   ;;
+send)
+   echo "${C_WHITE}Sending all packs...${C_DEFAULT}"
+   cpath="ControlPath=$(pwd)/mux-packs"
+   (
+     trap close_master SIGHUP SIGINT SIGTERM SIGKILL
+
+     ssh -nNf -o ControlMaster=yes -o "$cpath" $target_host
+     export GIT_SSH_COMMAND="ssh -o $cpath" 
+     echo "${C_GREEN}Uploading $sub...${C_DEFAULT}"
+     current_branch=$(get_current_branch)
+     git send-pack "$target_host:$target_directory" "$target_ref/$current_branch"
+     # Wait a bit for SSH to shutdonw the tunnel between each git command. Some
+     # server are setup to only allow one tunnel at a time.
+     sleep 0.5
+     for i in $submodules; do
+       IFS=","
+       set -- $i;
+       sub_git=$1
+       sub=$2
+       branch=$3
+       echo "${C_GREEN}Uploading $sub...${C_DEFAULT}"
+       [ -e extra/$sub ] && (cd extra/$sub; git send-pack "$target_host:$target_directory/extra/$sub" "$target_ref/$branch")
+       sleep 0.5
+     done
+
+     ssh -O exit -o ${cpath} ${target_host}
+   )
+   ;;
+pull)
+   echo "${C_WHITE}Pulling root...${C_DEFAULT}"
+   ( git pull 2>&1 ) | indent_out
+   for i in $submodules; do
+      IFS=","
+      set -- $i;
+      sub_git=$1
+      sub=$2
+      branch=$3
+      echo "${C_WHITE}Pulling ${sub_git}${C_DEFAULT} into extra/${sub}"
+      [ -e extra/$sub ] && (cd extra/$sub; git pull 2>&1 ) | indent_out
+   done
+   ;;
+push)
+   for i in $submodules; do
+      IFS=","
+      set -- $i;
+      sub_git=$1
+      sub=$2
+      branch=$3
+      echo "${C_WHITE}Pushing ${sub_git}${C_DEFAULT}..."
+      [ -e extra/$sub ] && (cd extra/$sub; git push origin ${branch} 2>&1 ) | indent_out
+   done
+   git push origin
+   ;;
+push_tags)
+   for i in $submodules; do
+      IFS=","
+      set -- $i;
+      sub_git=$1
+      sub=$2
+      branch=$3
+      echo "${C_WHITE}Pushing ${sub_git}${C_DEFAULT}..."
+      [ -e extra/$sub ] && (cd extra/$sub; git push --tags origin 2>&1 ) | indent_out
+   done
+   git push --tags origin
+   ;;
+purge)
+   for i in $submodules; do
+      IFS=","
+      set -- $i;
+      sub_git=$1
+      sub=$2
+      branch=$3
+      [ -e extra/$sub ] && (
+        cd extra/${sub};
+        if [ -z "$(git status --porcelain)" ]; then
+          # Working directory clean
+	  cd ..
+          echo "Purging ${sub}"
+	  rm -fr ${sub}
+        else
+          echo "Not empty. Skipping"
+        fi
+      )
+   done
+   ;;
+clone)
+  for i in $submodules; do
+      IFS=","
+      set -- $i;
+      sub_git=$1
+      sub=$2
+      if [[ $sub_git =~ ^bb:(.*)$ ]]; then
+	sub_git=${BASH_REMATCH[1]}
+      else
+        sub_git=bayesian_lss_team/${sub_git}.git
+      fi
+      echo "Cloning ${sub_git} into extra/${sub}"
+      (
+        cd extra/
+        if ! test -e ${sub}; then
+          git clone  ${prefix}${sub_git} ${sub} 
+        fi
+      ) 2>&1 | indent_out
+  done
+  ;;
+hooks)
+  ares_topdir=$(pwd)
+  echo "... setting up root"
+  setup_hooks ${ares_topdir}
+  for i in $submodules; do
+      IFS=","
+      set -- $i;
+      sub_git=$1
+      sub=$2
+      if [[ $sub_git =~ ^bb:(.*)$ ]]; then
+	sub_git=${BASH_REMATCH[1]}
+      else
+        sub_git=bayesian_lss_team/${sub_git}.git
+      fi
+      echo "... setting up ${sub}"
+      IFS=$'\n\t '
+      (
+        cd extra/${sub}
+        setup_hooks ${ares_topdir}
+      )
+  done
+  ;;
+update_copyright)
+  check_dirty .
+  python3 build_tools/gather_sources.py
+  for i in $submodules; do
+      IFS=","
+      set -- $i;
+      sub_git=$1
+      sub=$2
+      branch=$3
+      echo "Updating ${sub}"
+      check_dirty extra/${sub}
+      [ -e extra/$sub ] && (cd extra/${sub};  python3 ../../build_tools/gather_sources.py)
+  done
+  ;;
+update_indent)
+  if ! command -v clang-format; then
+    errormsg "Could not find clang-format in the PATH. It is required for reindentation."
+    exit 1  
+  fi
+  CF=$(which clang-format)
+  for i in $submodules; do
+      IFS=","
+      set -- $i;
+      sub_git=$1
+      sub=$2
+      branch=$3
+      if test -d extra/${sub}; then
+        find extra/${sub} -name "*.[ch]pp" | xargs ${CF} -style=file -i  
+      fi
+  done
+  ;;
+branch_set)
+  echo "Setting up branches for modules..."
+  for i in ${submodules}; do
+      IFS=","
+      set -- $i;
+      sub_git=$1
+      sub=$2
+      branch=$3
+      echo "-- Switching ${sub} to ${branch}"
+      if ! [ -e extra/$sub ]; then
+        echo "${C_RED}Directory extra/$sub does not exist. Failing.${C_DEFAULT}"
+        exit 1
+      fi
+      ( cd extra/${sub};
+	 if ! (git checkout ${branch} > /dev/null 2>&1); then
+            errormsg "Problem changing branch on ${sub}"
+            exit 1
+         fi
+      )
+  done
+  ;;
+report)
+  echo "GIT report"
+  for i in . extra/*; do
+    if test x$(basename $i) = xdemo; then
+      continue
+    fi
+    (
+      cd $i
+      h=$(git rev-parse HEAD)
+      mname=$(if [ "$i" == "." ]; then echo "root"; else echo $(basename $i); fi) 
+      status=`git status 2>&1 | tee`
+      dirty=`printf "${status}" 2> /dev/null | grep "modified:" &> /dev/null; echo "$?"`
+      code=""
+      if [ "$dirty" == "0" ] ; then
+        code='(?)'
+      fi
+      echo "- Module: ${mname}, Version${code}: ${h}"
+    )
+  done
+  ;;
+worktree)
+  base=$(pwd)
+  git worktree add "${branch_dir}" "${branch_work}" || abort "Invalid worktree arguments"
+  work_submodules=$(read_submodules ${branch_dir}/.aquila-modules)
+  for i in ${work_submodules}; do
+      IFS=","
+      set -- $i;
+      sub_git=$1
+      sub=$2
+      branch=$3
+      ( cd "extra/$sub"; git worktree add ${base}/${branch_dir}/extra/${sub} "${branch}" || abort "Invalid state" ) || abort "Could not setup ${branch} for ${sub}"
+  done
+  ;;
+status)
+  echo "Checking GIT status..."
+  echo
+  for i in . extra/*; do
+    if test x$(basename $i) = xdemo; then
+      continue
+    fi
+    (
+      cd $i
+      # https://stackoverflow.com/questions/1593051/how-to-programmatically-determine-the-current-checked-out-git-branch
+      branch_name="$(git symbolic-ref HEAD 2>/dev/null)" || branch_name="(unnamed branch)"     # detached HEAD
+      branch_name=${branch_name##refs/heads/}
+      # --
+      if [ "$i" == "." ]; then
+	printf "Root tree\t (branch ${branch_name}) :"
+      else
+        printf "Module $(basename $i)\t(branch ${branch_name}) :"
+      fi
+      status=`git status 2>&1 | tee`
+      dirty=`printf "${status}" 2> /dev/null | grep "modified:" &> /dev/null; echo "$?"`
+      untracked=`printf "${status}" 2> /dev/null | grep "Untracked files" &> /dev/null; echo "$?"`
+      ahead=`printf "${status}" 2> /dev/null | grep "Your branch is ahead of" &> /dev/null; echo "$?"`
+      newfile=`printf "${status}" 2> /dev/null | grep "new file:" &> /dev/null; echo "$?"`
+      renamed=`printf "${status}" 2> /dev/null | grep "renamed:" &> /dev/null; echo "$?"`
+      deleted=`printf "${status}" 2> /dev/null | grep "deleted:" &> /dev/null; echo "$?"`
+      code=$(
+      [ "$dirty" == "0" ] && echo "  dirty"
+      [ "$untracked" == "0" ] && echo "  untracked. Are you sure all files are under git supervision ?"
+      [ "$ahead" == "0" ] && echo "  ahead. Run git push ?"
+      [ "$newfile" == "0" ] && echo  "  newfile. Please git commit." 
+      [ "$renamed" == "0" ] && echo  "  renamed. Please git commit."
+      [ "$deleted" == "0" ] && echo  "  deleted. Please git commit.")
+      if [ "x$code" == "x" ]; then
+	 printf "$C_GREEN good. All clear. $C_DEFAULT\n"
+      else
+	 printf "$C_RED some things are not right. $C_DEFAULT\n"
+	 printf "${code}\n"
+      fi
+    )
+  done 
+  ;;
+esac
+# ===========================================
+
+
+if [ $operation != report ]; then
+  check_hooks
+fi