Sunday, January 9, 2011

[TECH] git encrypted backup script

Another free script! This one does encrypted space-efficient encrypted git backups from multiple repositories and working copies hosted on multiple machines to multiple destination directories on a single machine and/or an Amazon S3 bucket. It's called cmgeneral.sh, for "commit-master general." The details of operation are best understood by studying the script, but basically:
  • you need a modes/{modename} directory for each backup configuration. a backup configuration specifies what to back up from where and how to save the resulting archive. this directory must contain:
    • a blank file called .ddd1531eafda97524d852328d208469e, to guard against typos
    • a file called codename, that contains the name of the mode (usually the same as the directory name). this will be incorporated in the name of the final archive for identification.
    • a file called list, containing three words per line: a user name, a host name, and an absolute path leading to a git repository
    • a file called dest, containing, one-per-line, directories to save backup archives in. each such directory must contain a file called .ddd1531eafda97524d852328d208469e, to guard against typos
    • a soft link called j3cmd leading to the synchronize.sh script from an installation of jets3t (required for s3 backups only)
    • a file called j3env containing a bash expression to export JETS3T_HOME to point to the jets3t installation directory (required for s3 backups only)
    • a file called j3bin containing the name of an s3 bucket to save backup archives in (required for s3 backups only)
  • when invoking the script, specify the modename as a first argument, and optionally "s3" as a second argument to enable saving backup archives to s3.
  • the first time the script is invoked, it will walk you through generating an RSA keypair for encryption using gpg. since public-key cryptography is used, a passphrase is not needed for backup, only for recovery.
  • each archive contains the encryption keys as well as an embedded recovery script. the passphrase is required for recovery.
  • the backup does not save the full state of each repository. in particular, tags are not saved. it saves all commits, though, which is all I am personally interested in recovering. it is possible that I will improve the script to save tags as well; this seems to be somewhat nontrivial.
  • any questions? send me an email zerosum42 [AT] gmail [DOT] com
#!/bin/bash
# copyright (c) 2011 by andrei borac

ARG1="$1"
ARG2="$2"

sudo umount /tmp/cmg-ddd1531eafda97524d852328d208469e-*/.gnupg &> /dev/null
sudo umount /tmp/cmg-ddd1531eafda97524d852328d208469e-* &> /dev/null
sudo rmdir /tmp/cmg-ddd1531eafda97524d852328d208469e-* &> /dev/null

set -o errexit
set -o nounset
set -o pipefail

if [ ! -f modes/"$ARG1"/.ddd1531eafda97524d852328d208469e ]
then
  echo "sorry, invalid mode"
  exit 1
fi

XMODE="`readlink -f modes/"$ARG1"`"

STAMP="`date +%Yy%mm%dd`"
TMPWD=/tmp/cmg-ddd1531eafda97524d852328d208469e-"$STAMP"
mkdir -p "$TMPWD"
sudo mount -t tmpfs tmpfs "$TMPWD"
sudo chown "$USER":"$USER" "$TMPWD"
mkdir -p "$TMPWD"/.gnupg
sudo mount -t ramfs ramfs "$TMPWD"/.gnupg
sudo chown "$USER":"$USER" "$TMPWD"/.gnupg
chmod a-rwx,u+rwx "$TMPWD"/.gnupg
cd "$TMPWD"

if [ ! -f "$XMODE"/public.key ] || [ ! -f "$XMODE"/secret.key ]
then
  echo "when creating a keypair, use \"8, q, 4096, 1000y, y, cmgeneral,,, o, {pwd}\""
  echo -n "initialize keypair for this mode? (y/N) "
  read -s -n 1 YCHAR
  echo
  if [ "$YCHAR" == "y" ]
  then
    GPGOPT="--homedir .gnupg --keyring .gnupg/public.keyring --secret-keyring .gnupg/secret.keyring --no-default-keyring"
    gpg $GPGOPT --expert --gen-key
    gpg $GPGOPT --export            -a cmgeneral > "$XMODE"/public.key
    gpg $GPGOPT --export-secret-key -a cmgeneral > "$XMODE"/secret.key
  else
    exit 1
  fi
fi

if [ ! -f "$XMODE"/codename ]
then
  echo "echo codename > modes/$ARG1/codename"
  echo "the codename will suffix cmgeneral archives for easy identification"
  exit 1
fi

XCODE="`cat \"$XMODE\"/codename`"

XLIST="`cat \"$XMODE\"/list | tr '\t' ' ' | sed -e 's/^[ ]*//' -e 's/[ ]*$//' -e 's/[ ][ ]*/:/g' | tr '\n' '@' | sed -e 's/@@*/@/g' -e 's/^@*//g' -e 's/@*$//g'`""@"
echo "XLIST='$XLIST'"
XDEST="`cat \"$XMODE\"/dest | tr '\n' ' '`"
echo "XDEST='$XDEST'"

git init
XIDID=1000

while [ 1 ]
do
  XIDID=$((XIDID+1))
  
  XLINE="${XLIST%%@*}"
  XLIST="${XLIST#*@}"
  
  if [ "$XLINE" == "" ]
  then
    break
  fi
  
  XUSER="${XLINE%%:*}"
  XLINE="${XLINE#*:}"
  
  XHOST="${XLINE%%:*}"
  XLINE="${XLINE#*:}"
  
  XPATH="${XLINE%%:*}"
  XLINE="${XLINE#*:}"
  
  echo "XIDID='$XIDID'"
  echo "XUSER='$XUSER'"
  echo "XHOST='$XHOST'"
  echo "XPATH='$XPATH'"
  
  echo "$XIDID $XUSER $XHOST $XPATH" >> toc
  git remote add recover_"$XIDID" "$XUSER"@"$XHOST":"$XPATH"
  git fetch --no-tags recover_"$XIDID"
done

git fsck --full --strict
git repack -a -d -f --window=100 --depth=100 --window-memory=64m
git fsck --full --strict
git bundle create bundle.git --remotes
fakeroot tar -cf contents.tar toc bundle.git

cp "$XMODE"/public.key "$XMODE"/secret.key .
GPGOPT="--homedir .gnupg --keyring .gnupg/public.keyring --secret-keyring .gnupg/secret.keyring --no-default-keyring"
gpg $GPGOPT --import public.key
gpg $GPGOPT --import secret.key
gpg $GPGOPT --list-keys --with-colons --with-fingerprint | egrep fpr | sed -e 's/^fpr[:]*//g' | sed -e 's/$/6:/' | gpg $GPGOPT --import-ownertrust
gpg $GPGOPT --recipient cmgeneral --encrypt -z 0 < contents.tar > contents.tar.gpg
md5sum    contents.tar.gpg | egrep -o '^[0-9a-fA-F]{32}'  > contents.tar.gpg.md5sum
sha256sum contents.tar.gpg | egrep -o '^[0-9a-fA-F]{64}'  > contents.tar.gpg.sha256sum
sha512sum contents.tar.gpg | egrep -o '^[0-9a-fA-F]{128}' > contents.tar.gpg.sha512sum

cat > recover.sh << 'EOF'
#!/bin/bash

set -o errexit
set -o nounset
set -o pipefail

NONCE="`date +%Yy%mm%dd-%Hh%Mm%Ss-%ss-%Nn`"
mkdir recover-"$NONCE"
cd recover-"$NONCE"

mkdir .gnupg
sudo mount -t ramfs ramfs .gnupg
sudo chown "$USER":"$USER" .gnupg
chmod a-rwx,u+rwx .gnupg
GPGOPT="--homedir .gnupg --keyring .gnupg/public.keyring --secret-keyring .gnupg/secret.keyring --no-default-keyring"
gpg $GPGOPT --import ../public.key
gpg $GPGOPT --import ../secret.key
gpg $GPGOPT --list-keys --with-colons --with-fingerprint | egrep fpr | sed -e 's/^fpr[:]*//g' | sed -e 's/$/6:/' | gpg $GPGOPT --import-ownertrust
gpg $GPGOPT --decrypt < ../contents.tar.gpg > ../contents.tar
tar -C .. -xmf ../contents.tar

git init
git bundle unbundle ../bundle.git |\
while read SHA REF
do
  END="${REF#refs/remotes/}"
  git tag "${END//\//_}" "$SHA"
done

echo "use 'cd recover-*; git tag -l' to see what was recovered"
echo "remember, there is a table of contents in the 'toc' file"
echo "you won't see any files until you 'git merge' one of the recovered tags"
echo "good luck!"

sudo umount .gnupg
rmdir .gnupg
echo "+OK"
EOF
chmod a+x recover.sh

fakeroot tar -c recover.sh public.key secret.key contents.tar.gpg* > archive.tar

NONCE="`date +%Yy%mm%dd-%Hh%Mm%Ss-%ss-%Nn`"

for IDEST in $XDEST
do
  if [ ! -f "$IDEST"/.ddd1531eafda97524d852328d208469e ]
  then
    echo "sorry, magic file not found for destination '$IDEST', skipping"
  else
    cp archive.tar "$IDEST"/cmgeneral-"$NONCE"-"$XCODE".tar
  fi
done

if [ "$ARG2" == "s3" ] && [ -x "$XMODE"/s3cmd ] && [ -f "$XMODE"/s3cfg ] && [ -f "$XMODE"/s3bin ]
then
  mv archive.tar cmgeneral-"$NONCE"-"$XCODE".tar
  . "$XMODE"/j3env
  "$XMODE"/j3cmd --properties "$XMODE"/j3cfg UP --nodelete "`cat \"$XMODE\"/j3bin`" cmgeneral-*
fi

cd /
sudo umount "$TMPWD"/.gnupg
sudo umount "$TMPWD"
echo "+OK"

No comments:

Post a Comment