- Refactor Travis CI provision process to use common shell scripts
- Discard defunct Vagrant provision script in favor of common scripts
- Use a local mirror for faster provision in Vagrant
- Pre-install r-bioc-biocinstaller to avoid issues with broken biocLite.R script
Signed-off-by: Andreas Ringlstetter <andreas.ringlstetter@xxxxxxxxx>
---
.travis.yml | 70 +++---------------
Vagrantfile | 86 +++++++---------------
integration-scripts/README.md | 23 ++++++
integration-scripts/install_codeface_R.sh | 15 ++++
integration-scripts/install_codeface_node.sh | 8 ++
integration-scripts/install_codeface_python.sh | 11 +++
integration-scripts/install_common.sh | 18 +++++
integration-scripts/install_cppstats.sh | 18 +++++
integration-scripts/install_repositories.sh | 15 ++++
integration-scripts/setup_database.sh | 10 +++
.../test_codeface.sh | 4 +-
11 files changed, 156 insertions(+), 122 deletions(-)
create mode 100644 integration-scripts/README.md
create mode 100755 integration-scripts/install_codeface_R.sh
create mode 100755 integration-scripts/install_codeface_node.sh
create mode 100755 integration-scripts/install_codeface_python.sh
create mode 100755 integration-scripts/install_common.sh
create mode 100755 integration-scripts/install_cppstats.sh
create mode 100755 integration-scripts/install_repositories.sh
create mode 100755 integration-scripts/setup_database.sh
rename run_integration.sh => integration-scripts/test_codeface.sh (60%)
diff --git a/.travis.yml b/.travis.yml
index 47c849a..9037285 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,66 +8,14 @@ virtualenv:
env:
- CPPSTATS_VERSION=0.8.4
#sudo: false # use the new container-based Travis infrastructure
-before_install:
- #- sudo deb http://ppa.launchpad.net/marutter/rrutter/ubuntu precise main
- #- sudo deb-src http://ppa.launchpad.net/marutter/rrutter/ubuntu precise
main
- - sudo add-apt-repository -y ppa:marutter/rrutter
- - sudo add-apt-repository -y ppa:marutter/c2d4u
- - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E084DAB9
- - sudo apt-get update -qq
- - mysql -e "CREATE DATABASE codeface;" -uroot
- - mysql -e "CREATE USER 'codeface'@'localhost' IDENTIFIED BY 'codeface';"
-uroot
- - mysql -e "GRANT ALL PRIVILEGES ON * . * TO 'codeface'@'localhost';" -uroot
+before_install:
+ - sudo integration-scripts/install_repositories.sh
install:
- # R
- - sudo apt-get install r-base r-base-dev
- # Generic packages
- - sudo apt-get install sinntp texlive default-jdk mysql-common mysql-client
mysql-server python-dev exuberant-ctags nodejs npm git subversion libgles2-mesa
python-pip sloccount graphviz doxygen
- # develop packages
- - sudo apt-get install libxml2-dev libcurl4-openssl-dev xorg-dev libx11-dev
libgles2-mesa-dev libglu1-mesa-dev libmysqlclient-dev libcairo2-dev libxt-dev
libcairo2-dev libmysqlclient-dev
- # Devel packages required for Ubuntu 14.04
- - sudo apt-get install libpoppler-dev libpoppler-glib-dev
- # Devel packages required for python packages
- - sudo apt-get install libyaml-dev
- # install python requirements
- #- sudo pip install -r python_requirements.txt
- #- sudo apt-get install python-yaml python-progressbar
- #- sudo -H pip install python-ctags
- - pip install -r python_requirements.txt
- # install cppstats
- - wget
https://codeload.github.com/clhunsen/cppstats/tar.gz/v$CPPSTATS_VERSION -O
/tmp/cppstats.tar.gz
- - tar -xvf /tmp/cppstats.tar.gz
- - export CPPSTATS=$PWD/cppstats-$CPPSTATS_VERSION/
- - echo '#!/bin/bash' > $CPPSTATS/cppstats
- - echo "cd $CPPSTATS" >> $CPPSTATS/cppstats
- - echo "PYTHONPATH=\"\$PYTHONPATH:$CPPSTATS/lib\" ./cppstats.py \"\$@\"" >>
$CPPSTATS/cppstats
- - chmod +x $CPPSTATS/cppstats
- - export PATH=$PATH:$CPPSTATS/
- - wget http://sdml.info/lmcrs/srcML-Ubuntu12.04-64.tar.gz -O
/tmp/srcML.tar.gz
- - tar -xvf /tmp/srcML.tar.gz
- - cp -rf $PWD/srcML/* $CPPSTATS/lib/srcml/linux/
- # cppstats dependencies
- - sudo apt-get install astyle xsltproc libxml2 libxml2-dev python
python-libxml2 python-lxml python-notify python-lxml gcc
- - sudo apt-get install libarchive12:i386
- - sudo ln -s /usr/lib/i386-linux-gnu/libarchive.so.12
/usr/lib/i386-linux-gnu/libarchive.so.2
- # Install R dependencies
- #- sudo apt-get install r-cran-rgraphviz
- - sudo apt-get install r-cran-ggplot2 r-cran-tm r-cran-tm.plugin.mail
r-cran-optparse
- - sudo apt-get install r-cran-igraph r-cran-zoo r-cran-xts r-cran-lubridate
r-cran-xtable
- - sudo apt-get install r-cran-reshape r-cran-wordnet r-cran-stringr
r-cran-yaml r-cran-plyr
- - sudo apt-get install r-cran-scales r-cran-gridExtra r-cran-scales
r-cran-RMySQL r-cran-RJSONIO
- - sudo apt-get install r-cran-RCurl r-cran-mgcv r-cran-shiny r-cran-dtw
r-cran-httpuv r-cran-devtools
- - sudo apt-get install r-cran-png r-cran-rjson r-cran-lsa r-cran-testthat
r-cran-arules r-cran-data.table r-cran-ineq
- - sudo Rscript packages.minimal.R
- #- sudo Rscript packages.R
- # Install node dependencies
- #- sudo update-alternatives --install /usr/bin/node node /usr/bin/nodejs 99
- - cd id_service
- - npm install addressparser express js-yaml mysql body-parser
- - cd ..
- # Setup database
- - mysql -ucodeface -pcodeface < datamodel/codeface_schema.sql
- - sudo python setup.py -q install
+ - sudo integration-scripts/install_common.sh
+ - sudo integration-scripts/install_codeface_R.sh
+ - sudo integration-scripts/install_codeface_python.sh
+ - sudo integration-scripts/install_cppstats.sh
+ - sudo integration-scripts/setup_database.sh
+ - integration-scripts/install_codeface_node.sh
script:
- - ./run_integration.sh
-
+ - integration-scripts/test_codeface.sh
diff --git a/Vagrantfile b/Vagrantfile
index fac1144..64d21cb 100755
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -4,67 +4,18 @@
# Copyright Roger Meier <roger@xxxxxxxxxxxxxxxxx>
# SPDX-License-Identifier: Apache-2.0 BSD-2-Clause GPL-2.0+ MIT WTFPL
-$build_and_test = <<SCRIPT
-echo "Provisioning system to compile, test and develop."
+$build = <<SCRIPT
cd /vagrant
-sudo -E apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E084DAB9
-echo deb http://cran.r-project.org/bin/linux/ubuntu precise/ > /tmp/r-project
-sudo mv /tmp/r-project /etc/apt/sources.list.d/r-project.list
+integration-scripts/install_repositories.sh
+integration-scripts/install_common.sh
+integration-scripts/install_codeface_R.sh
+integration-scripts/install_codeface_node.sh
+integration-scripts/install_codeface_python.sh
-sudo apt-get update -qq -y
-sudo DEBIAN_FRONTEND=noninteractive apt-get -qq -y -o
Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" upgrade
+integration-scripts/install_cppstats.sh
-# GNU R
-sudo apt-get -qq -y install r-base r-base-dev
-
-# MySQL with vagrant password
-sudo debconf-set-selections <<< 'mysql-server mysql-server/root_password
password vagrant'
-sudo debconf-set-selections <<< 'mysql-server mysql-server/root_password_again
password vagrant'
-sudo apt-get -y install mysql-server mysql-common mysql-client
-
-# setup Database
-sudo mysql --user=root --password=vagrant -e "create database codeface; GRANT
ALL PRIVILEGES ON codeface.* TO codeface@localhost IDENTIFIED BY 'codeface'"
-sudo mysql -ucodeface -pcodeface < datamodel/codeface_schema.sql
-
-
-# Generic packages
-sudo apt-get -qq -y install python-mysqldb sinntp texlive default-jdk \
- python-dev \
- exuberant-ctags git subversion \
- libgles2-mesa python-pip sloccount
-
-# Devel packages required to build the R packages below from source
-sudo apt-get -qq -y install libxml2-dev libcurl4-openssl-dev xorg-dev \
- libx11-dev libgles2-mesa-dev libglu1-mesa-dev \
- libmysqlclient-dev libcairo2-dev libxt-dev \
- libcairo2-dev libmysqlclient-dev
-
-# Devel packages required for python packages
-sudo apt-get -qq -y install libyaml-dev
-sudo -E pip install pyyaml progressbar python-ctags
-
-# install a recent node.js version
-sudo apt-get install -qq -y python-software-properties python g++ make
-sudo add-apt-repository -y ppa:chris-lea/node.js
-sudo apt-get update -qq -y
-sudo apt-get install -qq -y nodejs
-sudo npm install addressparser express js-yaml mysql -g
-
-# install R packages
-sudo R CMD javareconf
-sudo cp .Rprofile ~/.Rprofile
-sudo R --save < packages.R
-
-sudo python setup.py develop --user
-sudo -E npm install -g \
- https://github.com/JohannesEbke/shiny-server/archive/no-su.tar.gz
-shiny-server shiny-server.config &
-
-# TODO: setup the missing stuff
-
-echo "provisioning of Codeface done."
-echo "see http://siemens.github.io/codeface/ for further info"
+integration-scripts/setup_database.sh
SCRIPT
Vagrant.configure("2") do |config|
@@ -80,7 +31,24 @@ Vagrant.configure("2") do |config|
config.vm.network :forwarded_port, guest: 8081, host: 8081
- # call the script
- config.vm.provision :shell, :inline => $build_and_test
+ config.vm.provision "fix-no-tty", type: "shell" do |s|
+ s.privileged = true
+ s.inline = "sed -i '/tty/!s/mesg n/tty -s \\&\\& mesg n/' /root/.profile"
+ end
+
+ config.vm.provision "local-mirror", type: "shell" do |s|
+ s.privileged = true
+ s.inline = "sed -i
's|http://[a-z\.]*\.ubuntu\.com/ubuntu|mirror://mirrors\.ubuntu\.com/mirrors\.txt|'
/etc/apt/sources.list"
+ end
+
+ config.vm.provision "build", type: "shell" do |s|
+ s.privileged = false
+ s.inline = $build
+ end
+
+ config.vm.provision "test", type: "shell" do |s|
+ s.privileged = false
+ s.inline = "cd /vagrant && integration-scripts/test_codeface.sh"
+ end
end
diff --git a/integration-scripts/README.md b/integration-scripts/README.md
new file mode 100644
index 0000000..d13d809
--- /dev/null
+++ b/integration-scripts/README.md
@@ -0,0 +1,23 @@
+Integration scripts
+===================
+
+Common scripts used by Vagrant and Travis CI to provision the VM required for
+integration testing. The working directory for all scripts must be the
codeface root directory.
+
+The scripts are meant to be run in a Ubuntu 12.04 LTS 64bit environment.
+
+Travis CI
+---------
+No further steps required.
+
+Vagrant
+-------
+`vagrant up` Create a new VM and execute all provision scripts. If the VM
already
+exists, provisioning is skipped and the existing VM is started instead.
+
+`vagrant ssh` Open a SSH session with the VM. Codeface is per default mounted
+from the host to `/vagrant` and owned by the user `vagrant`.
+
+`vagrant halt` Stop a running VM.
+
+`vagrant destroy` Destroy the VM and delete from file system.
\ No newline at end of file
diff --git a/integration-scripts/install_codeface_R.sh
b/integration-scripts/install_codeface_R.sh
new file mode 100755
index 0000000..58ebf04
--- /dev/null
+++ b/integration-scripts/install_codeface_R.sh
@@ -0,0 +1,15 @@
+#!/bin/sh
+
+echo "Providing R libraries"
+
+sudo DEBIAN_FRONTEND=noninteractive apt-get -qqy install r-base r-base-dev
r-cran-ggplot2 r-cran-tm \
+ r-cran-tm.plugin.mail r-cran-optparse r-cran-igraph r-cran-zoo
r-cran-xts \
+ r-cran-lubridate r-cran-xtable r-cran-reshape r-cran-wordnet \
+ r-cran-stringr r-cran-yaml r-cran-plyr r-cran-scales r-cran-gridExtra \
+ r-cran-scales r-cran-RMySQL r-cran-RJSONIO r-cran-RCurl r-cran-mgcv \
+ r-cran-shiny r-cran-dtw r-cran-httpuv r-cran-png \
+ r-cran-rjson r-cran-lsa r-cran-testthat r-cran-arules r-cran-data.table
\
+ r-cran-ineq libx11-dev libssh2-1-dev r-bioc-biocinstaller
+
+sudo Rscript packages.R
+
diff --git a/integration-scripts/install_codeface_node.sh
b/integration-scripts/install_codeface_node.sh
new file mode 100755
index 0000000..ceb70c2
--- /dev/null
+++ b/integration-scripts/install_codeface_node.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+echo "Providing id_service"
+
+cd id_service
+npm install
+cd ..
+
diff --git a/integration-scripts/install_codeface_python.sh
b/integration-scripts/install_codeface_python.sh
new file mode 100755
index 0000000..d9fd495
--- /dev/null
+++ b/integration-scripts/install_codeface_python.sh
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+echo "Providing codeface python"
+
+sudo pip install --upgrade -q setuptools mock
+
+# Only development mode works
+# install fails due to R scripts accessing unbundled resources!
+# TODO Fix the R scripts
+sudo python2.7 setup.py -q develop
+
diff --git a/integration-scripts/install_common.sh
b/integration-scripts/install_common.sh
new file mode 100755
index 0000000..b5ad849
--- /dev/null
+++ b/integration-scripts/install_common.sh
@@ -0,0 +1,18 @@
+#!/bin/sh
+
+echo "Providing common binaries and libraries"
+
+echo "mysql-server mysql-server/root_password password" | sudo
debconf-set-selections
+echo "mysql-server mysql-server/root_password_again password" | sudo
debconf-set-selections
+
+sudo DEBIAN_FRONTEND=noninteractive apt-get -qqy install sinntp texlive
default-jdk \
+ mysql-common mysql-client \
+ mysql-server python-dev exuberant-ctags nodejs git subversion \
+ sloccount graphviz doxygen libxml2-dev libcurl4-openssl-dev \
+ libmysqlclient-dev libcairo2-dev libxt-dev libcairo2-dev
libmysqlclient-dev \
+ astyle xsltproc libxml2 libxml2-dev python build-essential libyaml-dev\
+ gfortran python-setuptools python-pkg-resources python-numpy
python-matplotlib \
+ python-libxml2 python-lxml python-notify python-lxml gcc libarchive12
python-pip \
+ libxml2-dev libcurl4-openssl-dev xorg-dev libx11-dev libgles2-mesa-dev \
+ libglu1-mesa-dev libxt-dev libpoppler-dev libpoppler-glib-dev
python-mock
+
diff --git a/integration-scripts/install_cppstats.sh
b/integration-scripts/install_cppstats.sh
new file mode 100755
index 0000000..2ad1bd6
--- /dev/null
+++ b/integration-scripts/install_cppstats.sh
@@ -0,0 +1,18 @@
+#!/bin/sh
+
+export CPPSTATS_VERSION=0.8.4
+
+echo "Providing cppstats $CPPSTATS_VERSION"
+
+wget --quiet
https://codeload.github.com/clhunsen/cppstats/tar.gz/v$CPPSTATS_VERSION -O
/tmp/cppstats.tar.gz
+tar -xvf /tmp/cppstats.tar.gz
+export CPPSTATS=$PWD/cppstats-$CPPSTATS_VERSION/
+echo '#!/bin/bash' > $CPPSTATS/cppstats
+echo "cd $CPPSTATS" >> $CPPSTATS/cppstats
+echo "PYTHONPATH=\"\$PYTHONPATH:$CPPSTATS/lib\" ./cppstats.py \"\$@\"" >>
$CPPSTATS/cppstats
+chmod +x $CPPSTATS/cppstats
+wget --quiet http://sdml.info/lmcrs/srcML-Ubuntu12.04-64.tar.gz -O
/tmp/srcML.tar.gz
+tar -xvf /tmp/srcML.tar.gz
+cp -rf $PWD/srcML/* $CPPSTATS/lib/srcml/linux/
+
+sudo ln -sf $CPPSTATS/cppstats /usr/local/bin/cppstats
diff --git a/integration-scripts/install_repositories.sh
b/integration-scripts/install_repositories.sh
new file mode 100755
index 0000000..75c398d
--- /dev/null
+++ b/integration-scripts/install_repositories.sh
@@ -0,0 +1,15 @@
+#!/bin/sh
+
+sudo apt-get update -qq
+sudo DEBIAN_FRONTEND=noninteractive apt-get -qqy install
software-properties-common python-software-properties
+
+echo "Adding R cran repositories"
+sudo add-apt-repository -y ppa:marutter/rrutter
+sudo add-apt-repository -y ppa:marutter/c2d4u
+sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E084DAB9
+
+echo "Adding node.js repository"
+sudo add-apt-repository -y ppa:chris-lea/node.js
+
+sudo apt-get update -qq
+
diff --git a/integration-scripts/setup_database.sh
b/integration-scripts/setup_database.sh
new file mode 100755
index 0000000..bc617ad
--- /dev/null
+++ b/integration-scripts/setup_database.sh
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+echo "Providing codeface database"
+
+sudo mysql -e "CREATE DATABASE codeface;" -uroot
+sudo mysql -e "CREATE USER 'codeface'@'localhost' IDENTIFIED BY 'codeface';"
-uroot
+sudo mysql -e "GRANT ALL PRIVILEGES ON * . * TO 'codeface'@'localhost';" -uroot
+
+mysql -ucodeface -pcodeface < datamodel/codeface_schema.sql
+
diff --git a/run_integration.sh b/integration-scripts/test_codeface.sh
similarity index 60%
rename from run_integration.sh
rename to integration-scripts/test_codeface.sh
index 2406ad7..6e58fa0 100755
--- a/run_integration.sh
+++ b/integration-scripts/test_codeface.sh
@@ -5,7 +5,7 @@ node id_service.js ../codeface.conf &
node_job=$!
cd ..
-PYTHONPATH=$PWD ./codeface/runCli.py test -c codeface.conf
+codeface test -c codeface.conf
codeface_exit=$?
kill $node_job
-exit $codeface_exit
\ No newline at end of file
+exit $codeface_exit
--
2.5.1
--------------060907030204000203070104--
