[4.5.X] Refactor DB cache management in runtests and runmigration

bin/db.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+db_restore() {
+  DUMP=$1
+  DB_NAME=$2
+
+  echo "Removing database '$DB_NAME'"
+  dropdb $DB_NAME --if-exists
+  echo "Create database '$DB_NAME'"
+  createdb -O $DB_USER $DB_NAME
+
+  if [ -f "$DUMP" ]; then
+    echo "🐘 🐘 Database dump $DUMP found 🐘 🐘"
+    echo "Restore Database dump from $DUMP 📦⮕ 🐘"
+    psql -q -o /dev/null -d $DB_NAME -f "$DUMP"
+    psql -d $DB_NAME -P pager=off -c "SELECT name as installed_module FROM ir_module_module WHERE state = 'installed' ORDER BY name"
+    return 1
+  else
+    echo "No dump found matching"
+    return 0
+  fi
+}
+
+db_save() {
+  DB_NAME=$1
+  DUMP=$2
+  if [ ! -z "$DUMP" ]; then
+    echo "Dumping $DB_NAME into $DUMP 🐘⮕ 📦"
+    mkdir -p $(dirname $DUMP)
+    pg_dump -Fp -d $DB_NAME -O -f "$DUMP"
+    ls $DUMP
+  fi
+}

bin/runmigration

@@ -11,8 +11,6 @@
 #
 # And finally make a database dump if none exists for current VERSION.
 #
-# TODO: store cache on S3 to store one DB per version
-#
 # Environment variables:
 #
 # CREATE_DB_CACHE:
@@ -34,49 +32,50 @@
 #   steps on top of it.
 set -e
 
+source db.sh
 wait_postgres.sh
-CACHE_DIR=/.cachedb
 
-echo $CACHE_DIR
+CREATE_DB_CACHE=${CREATE_DB_CACHE:=false}
+LOAD_DB_CACHE=${LOAD_DB_CACHE:=true}
 
-VERSION=$(cat /odoo/VERSION)
+CACHE_DIR=${CACHE_DIR:=/tmp/cachedb}
+DB_NAME=${DB_NAME:=odoo_mig}
 
-CACHED_DUMP="$CACHE_DIR/odoo_sample_$VERSION.dmp"
+VERSION=$(cat /odoo/VERSION)
 
 if [ "$LOAD_DB_CACHE" != "false" ]; then
 
-    # If we want to run the migration steps on top of a previous dump
-    # useful when odoo/VERSION was edited
-    if [ -n "$MIG_LOAD_VERSION_CEIL" ]; then
-        echo "New version - Searching for previous version dump 🔭"
-        if [ -d "$CACHE_DIR" ]; then
-            # Filter dumps of higher releases
-            export MAX_DUMP="$CACHE_DIR/odoo_sample_${MIG_LOAD_VERSION_CEIL}.dmp"
-            CACHED_DUMP=$(ls -v $CACHE_DIR/odoo_sample_*.dmp | awk '$0 < ENVIRON["MAX_DUMP"]' | tail -n1)
-        else
-            echo "No cached migration sample dump found"
-        fi
+  # If we want to run the migration steps on top of a previous dump
+  # useful when odoo/VERSION was edited
+  if [ -n "$MIG_LOAD_VERSION_CEIL" ]; then
+    echo "New version - Searching for previous version dump 🔭"
+    if [ -d "$CACHE_DIR" ]; then
+      # Filter dumps of higher releases
+      export MAX_DUMP="$CACHE_DIR/odoo_sample_${MIG_LOAD_VERSION_CEIL}.dmp"
+      CACHED_DUMP=$(ls -v $CACHE_DIR/odoo_sample_*.dmp || true | awk '$0 < ENVIRON["MAX_DUMP"]' | tail -n1)
     fi
+    if [ -n "$CACHED_DUMP" ]; then
+      echo "No cached migration sample dump found"
+    fi
+  fi
 else
-    echo "Dump cache load disabled."
+  echo "DB cache load disabled."
 fi
 
-if [ "$LOAD_DB_CACHE" != "false" -a -f "$CACHED_DUMP" ]; then
-    echo "🐘 🐘 Database dump ${CACHED_DUMP} found 🐘 🐘"
-    echo "Restore Database dump from cache 📦⮕ 🐘"
-    createdb -O $DB_USER $DB_NAME
-    psql -q -o /dev/null -f "$CACHED_DUMP"
+if [ "$LOAD_DB_CACHE" != "false" ]; then
+  db_restore "$CACHED_DUMP" "$DB_NAME"
+  if [ $? = 1 ]; then
     echo "Do migration on top of restored dump"
-else
+  else
     echo "Do migration from scratch 🐢 🐢 🐢"
+  fi
 fi
 
 gosu odoo migrate
 
+DUMP="$CACHE_DIR/odoo_sample_$VERSION.dmp"
 # Create a dump if none exist for the current VERSION
-if [ "$CREATE_DB_CACHE" == "true" -a ! -f "$CACHED_DUMP" ]; then
-    echo "Save DB to cache $CACHED_DUMP 🐘⮕ 📦"
-    mkdir -p "$CACHE_DIR"
-    pg_dump -Fp -O -f "$CACHED_DUMP"
-    ls -l $CACHED_DUMP
+if [ "$CREATE_DB_CACHE" == "true" -a ! -f $DUMP ]; then
+  mkdir -p "$CACHE_DIR"
+  db_save $DB_NAME $DUMP
 fi

bin/runtests

@@ -27,63 +27,61 @@
 set -e
 
 # TODO: if we are not in TRAVIS, make a template then run tests on a copy
-
+source db.sh
 wait_postgres.sh
 LOCAL_SRC_DIR=/odoo/local-src
 case "$ODOO_VERSION" in
-    "7.0")          ODOO_BIN_PATH=/odoo/src/openerp-server ;;
-    "8.0"|"9.0")    ODOO_BIN_PATH=/odoo/src/odoo.py ;;
-    *)              ODOO_BIN_PATH=/odoo/src/odoo-bin ;;
+"7.0") ODOO_BIN_PATH=/odoo/src/openerp-server ;;
+"8.0" | "9.0") ODOO_BIN_PATH=/odoo/src/odoo.py ;;
+*) ODOO_BIN_PATH=/odoo/src/odoo-bin ;;
 esac
-CACHE_DIR=/.cachedb
 
-if [ -z $1 ]
-then
-  LOCAL_ADDONS=$(find ${LOCAL_SRC_DIR}/* -maxdepth 0 -type d -and -not -name server_environment_files -printf "%f\n"\
-                | awk -vORS=, '{print $1}'\
-                | sed 's/,$/\n/')
+if [ -z $1 ]; then
+  LOCAL_ADDONS=$(find ${LOCAL_SRC_DIR}/* -maxdepth 0 -type d -and -not -name server_environment_files -printf "%f\n" |
+    awk -vORS=, '{print $1}' |
+    sed 's/,$/\n/')
 else
   LOCAL_ADDONS=$1
 fi
 
-DEPS_ADDONS=$(list_dependencies.py "$LOCAL_ADDONS")
+CREATE_DB_CACHE=${CREATE_DB_CACHE:=false}
+LOAD_DB_CACHE=${LOAD_DB_CACHE:=true}
 
-DB_NAME_TEST=${DB_NAME}_test
+LOCAL_SRC_DIR=${LOCAL_SRC_DIR:=/odoo/local-src}
+CACHE_DIR=${CACHE_DIR:=/tmp/cachedb}
 
+DEPS_ADDONS=$(list_dependencies.py "$LOCAL_ADDONS")
 
-echo "Create database"
-createdb -O $DB_USER ${DB_NAME_TEST}
+DB_NAME=${DB_NAME:=odoo_test}
 
 if [[ ! -z "$SUBS_MD5" ]]; then
-    CACHED_DUMP="$CACHE_DIR/odoo_test_$SUBS_MD5.dmp"
+  DUMP="$CACHE_DIR/odoo_test_$SUBS_MD5.dmp"
 fi
 
 echo "Submodule addons MD5 is: $SUBS_MD5"
 
-if [ "$LOAD_DB_CACHE" != "false" -a -f "$CACHED_DUMP" ]; then
-    echo "🐘 🐘 Database dump ${CACHED_DUMP} found 🐘 🐘"
-    echo "Restore Database dump from cache matching MD5 📦⮕ 🐘"
-    psql -q -o /dev/null -d $DB_NAME_TEST -f "$CACHED_DUMP"
-    psql -d $DB_NAME_TEST -P pager=off -c "SELECT name as installed_module FROM ir_module_module WHERE state = 'installed' ORDER BY name"
+DUMP_LOADED=0
+if [ "$LOAD_DB_CACHE" != "false" ]; then
+  db_restore "$DUMP" "$DB_NAME"
+  DUMP_LOADED=$?
+  if [ $DUMP_LOADED = 1 ]; then
+    echo "Use restored dump with official/OCA modules"
+  else
+    echo "Reinstall all addons from scratch 🐢 🐢 🐢"
+  fi
 else
-    if [ "$LOAD_DB_CACHE" == "false" ]; then
-        echo "Dump cache load disabled."
-    else
-        echo "No cached dump found matching MD5 🐢 🐢 🐢"
-    fi
-    echo "🔨🔨 Install official/OCA modules 🔨🔨"
-    gosu odoo odoo --stop-after-init --workers=0 --database $DB_NAME_TEST --log-level=warn --without-demo="" --db-filter=$DB_NAME_TEST -i ${DEPS_ADDONS}
-    if [ "$CREATE_DB_CACHE" == "true" -a ! -z "$CACHED_DUMP" ]; then
-        echo "Generate dump $CACHED_DUMP into cache 🐘⮕ 📦"
-        mkdir -p "$CACHE_DIR"
-        pg_dump -Fp -d $DB_NAME_TEST -O -f "$CACHED_DUMP"
-    fi
+  echo "DB cache load disabled."
 fi
-echo "🔧🔧 Install local-src modules 🔧🔧"
-gosu odoo odoo --stop-after-init --workers=0 --database $DB_NAME_TEST --log-level=warn --without-demo="" --db-filter=$DB_NAME_TEST -i ${LOCAL_ADDONS}
 
-export COVERAGE_FILE=/home/odoo/.coverage
-gosu odoo coverage run --source="${LOCAL_SRC_DIR}" "${ODOO_BIN_PATH}" --stop-after-init --workers=0 --database $DB_NAME_TEST --test-enable --log-level=test --log-handler=":INFO" --db-filter=$DB_NAME_TEST -u ${LOCAL_ADDONS}
-dropdb ${DB_NAME_TEST}
+if [ "$DUMP_LOADED" = 0 ]; then
+  echo "🔨🔨 Install official/OCA modules 🔨🔨"
+  odoo --stop-after-init --workers=0 --database $DB_NAME --log-level=warn --without-demo="" --db-filter=$DB_NAME -i ${DEPS_ADDONS}
+  if [ "$CREATE_DB_CACHE" != "false" ]; then
+    db_save $DB_NAME $DUMP
+  fi
+fi
+echo "🔧🔧 Install local-src modules 🔧🔧"
+coverage run --source="${LOCAL_SRC_DIR}" "${ODOO_BIN_PATH}" --stop-after-init --workers=0 --database $DB_NAME --test-enable --log-level=test --log-handler=":INFO" --without-demo="" --db-filter=$DB_NAME -i ${LOCAL_ADDONS}
 
+dropdb ${DB_NAME}
 coverage report -m

test.sh

@@ -88,8 +88,6 @@ echo "        post:" >>odoo/migration.yml
 echo "          - anthem songs.install.demo::create_partners" >>odoo/migration.yml
 docoruncmd odoo dropdb odoodb
 
-
-
 echo '>>> * run unit tests with runtests'
 docoruntests -e LOAD_DB_CACHE="false" -e CREATE_DB_CACHE="false"
 
@@ -99,4 +97,3 @@ docoruntests -e CREATE_DB_CACHE="true" -e SUBS_MD5=testcache
 echo '>>> * run unit tests with runtests and re-use a dump'
 docoruntests -e LOAD_DB_CACHE="true" -e SUBS_MD5=testcache
 docodown
-