jenkins: add llnode pipeline and shell scripts (#1025)

* jenkins: add llnode pipeline and shell scripts

The llnode CI isn't working right now, but hopefully by committing this
we can then iterate in GitHub.

Refs: #777

Responsible-commiter: Refael Ackermann <[email protected]>
Reviewed-By: Matheus Marchini <[email protected]>

Author: gibfahn

jenkins/pipelines/llnode-pipeline.jenkinsfile

@@ -0,0 +1,43 @@
+#!/usr/bin/env groovy
+
+properties([
+  parameters([
+    string(name: 'GIT_REPO', defaultValue: 'nodejs/llnode', description: 'Fetch from this repo (e.g. gibfahn/llnode)'),
+    string(name: 'GIT_BRANCH', defaultValue: 'master', description: 'Branch to test (e.g. master, refs/pull/999/head).'),
+    string(name: 'NODE_VERSIONS', defaultValue: 'v6 v8 v9 v10 nightly canary', description: 'Space separated list of NODE_VERSIONS to pass to llnode-continuous-integration.'),
+    string(name: 'LLDB_VERSIONS', defaultValue: '3.9 4.0 5.0 6.0', description: 'Space separated list of LLDB_VERSIONS to pass to llnode-continuous-integration.'),
+    string(name: 'MACHINES', defaultValue: 'all', description: '''Can be "all" or a space-separated list of nodes, eg:
+      debian8-64 fedora22 fedora23 osx1010 ppcle-ubuntu1404 ubuntu1204-64 ubuntu1404-64 ubuntu1604-64 rhel72-s390x aix61-ppc64 ppcbe-ubuntu1404 smartos16-64 smartos15-64 win10 win2012r2'''),
+  ]),
+])
+
+if (!params['GIT_REPO']) { error("You didn't define a GIT_REPO.") }
+
+def pr = [] // Mutable parameter set to pass to jobs. I know this is crazy.
+String printParams = '' // String of params to print
+for (param in params) {
+    printParams += param.toString() + '\n'
+    if (param.key != 'NODE_VERSIONS') {
+        pr.push(string(name: param.key, value: param.value))
+    }
+}
+println "\n### BUILD PARAMETERS ###\n${printParams}"
+
+stage('Test llnode') {
+    def nodeVersions = params['NODE_VERSIONS'].split()
+    def lldbVersions = params['LLDB_VERSIONS'].split()
+    def buildJobs = [:]
+    for (int i = 0; i < nodeVersions.length; i++) {
+        int nodeIndex = i // locally scoped copy.
+        for (int j = 0; j < lldbVersions.length; j++) {
+            int lldbIndex = j // locally scoped copy.
+            def p = pr.collect() // local copy of params
+            p.push(string(name: 'NODE_VERSION', value: nodeVersions[nodeIndex]))
+            p.push(string(name: 'LLDB_VERSION', value: lldbVersions[lldbIndex]))
+            buildJobs["Node.js ${nodeVersions[nodeIndex]} lldb ${lldbVersions[lldbIndex]}"] = { build(job: 'llnode-continuous-integration', parameters: p) }
+        }
+    }
+
+    println buildJobs
+    parallel(buildJobs)
+}

jenkins/scripts/common/colors.sh

@@ -0,0 +1,61 @@
+# Helper vars for printing in color. Use with:
+# echo -e "${RED}foo${NC}" # Echo foo in red, then reset color.
+# printf "${BBLUE}foo${NC}" # Echo foo in bright blue, then reset color.
+
+# Include (source) with this line:
+# . $(dirname $0)/helpers/colors.sh # Load helper script from gcfg/helpers.
+
+# Color escape codes, use with . Don't change \033 to \e, that doesn't work on
+# macOS (\x1B instead).
+
+# I've kept the standard names, even though e.g. WHITE isn't white (use BWHITE).
+# There are also a bajillion more combos (e.g. BGREDFGBLUE) with different
+# FG/BG color combos, but I haven't needed these yet.
+
+case $- in *x*) xSet=true ;; esac # Note whether -x was set before.
+set +x
+
+# Reset color:
+NC='\033[0m' # No Color.
+
+# Basic colors:
+BLACK='\033[0;30m'      # Black.
+RED='\033[0;31m'        # Red.
+GREEN='\033[0;32m'      # Green.
+YELLOW='\033[0;33m'     # Yellow.
+BLUE='\033[0;34m'       # Blue.
+MAGENTA='\033[0;35m'    # Magenta (purple).
+CYAN='\033[0;36m'       # Light blue.
+WHITE='\033[0;37m'      # Light grey.
+
+# Bright colors:
+BBLACK='\033[1;30m'     # Bright black (dark grey).
+BRED='\033[1;31m'       # Bright Red.
+BGREEN='\033[1;32m'     # Bright Green.
+BYELLOW='\033[1;33m'    # Bright Yellow.
+BBLUE='\033[1;34m'      # Bright Blue.
+BMAGENTA='\033[1;35m'   # Bright Magenta (pink).
+BCYAN='\033[1;36m'      # Bright Light blue.
+BWHITE='\033[1;37m'     # Bright white (white).
+
+# Background colors:
+BGBLACK='\033[0;40m'    # Background Black.
+BGRED='\033[0;41m'      # Background Red.
+BGGREEN='\033[0;42m'    # Background Green.
+BGYELLOW='\033[0;43m'   # Background Yellow.
+BGBLUE='\033[0;44m'     # Background Blue.
+BGMAGENTA='\033[0;45m'  # Background Magenta (purple).
+BGCYAN='\033[0;46m'     # Background Light blue.
+BGWHITE='\033[0;47m'    # Background Light grey.
+
+# Bright background colors:
+BGBBLACK='\033[1;40m'   # Background Bright Black (dark grey).
+BGBRED='\033[1;41m'     # Background Bright Red.
+BGBGREEN='\033[1;42m'   # Background Bright Green.
+BGBYELLOW='\033[1;43m'  # Background Bright Yellow.
+BGBBLUE='\033[1;44m'    # Background Bright Blue.
+BGBMAGENTA='\033[1;45m' # Background Bright Magenta (pink).
+BGBCYAN='\033[1;46m'    # Background Bright Light blue.
+BGBWHITE='\033[1;47m'   # Background Bright white.
+
+[ "$xSet" ] && set -x # If -x was set before, restore it.

jenkins/scripts/common/getLLDB.sh

@@ -0,0 +1,31 @@
+#!/bin/bash -ex
+
+DOWNLOAD_DIR="http://apt.llvm.org/xenial/pool/main/l/llvm-toolchain-$LLDB_VERSION/"
+
+rm -rf lldb-bin/
+
+# Calculate os and arch.
+os="$(uname | tr '[:upper:]' '[:lower:]')"
+arch=$(uname -m)
+
+case $os in
+  linux) echo "$os found. Assuming Ubuntu 16.04...";;
+  darwin) exit 0;;
+  *) echo "$os CI is not supported yet"; exit 1;;
+esac
+
+case $arch in
+  x86_64) arch=amd64;;
+esac
+
+files=$(curl --compressed -L -s "$DOWNLOAD_DIR" | grep "^<a href=" | cut -d \" -f 2)
+debFilename=$(echo "$files" | grep "^lldb-$LLDB_VERSION.*$arch.deb")
+debFilename2=$(echo "$files" | grep "^libllvm$LLDB_VERSION.*$arch.deb")
+
+mkdir lldb-bin
+
+dpkg-deb -xv <(curl -L -s "${DOWNLOAD_DIR}${debFilename}") lldb-bin/
+dpkg-deb -xv <(curl -L -s "${DOWNLOAD_DIR}lib${debFilename}") lldb-bin/
+dpkg-deb -xv <(curl -L -s "${DOWNLOAD_DIR}${debFilename2}") lldb-bin/
+
+ln -s $(pwd)/lldb-bin/usr/bin/lldb-${LLDB_VERSION} $(pwd)/lldb-bin/usr/bin/lldb

jenkins/scripts/common/getNode.sh

@@ -0,0 +1,80 @@
+#!/bin/bash -ex
+
+# This script downloads the relevant Node binary for the arch from nodejs.org.
+# Download the latest v10/v9/v8/v6/v4 build from nodejs.org
+
+. "$(dirname $0)"/colors.sh # Load colors.
+
+# Help text to print on error.
+sep="${MAGENTA}:${NC}" # Separator to use in help text.
+helpText="Download the node binary for your machine from nodejs.org.
+  ${CYAN}USAGE:${NC}
+  $0 8.9.1
+
+  8.9.0            $sep Node to download (8.9.0, master, 8, 8.9.0-nightly, latest, 6.2.)
+                        Inexact values get most recent from range."
+
+################################################################################
+## Parse parameters
+
+unset NODE_VERSION
+while getopts ":h" option; do
+  case "${option}" in
+     h) echo -e "$helpText"; exit 0;;
+    \?) echo "Invalid option -$OPTARG."; exit 1 ;;
+     :) echo "Option -$OPTARG takes a parameter."; exit 1 ;;
+     *) echo "Script doesn't catch the parameter at $OPTIND, $OPTARG."; exit 1 ;;
+  esac
+done
+shift $((OPTIND-1))
+
+error() { echo -e "${BRED}ERROR:${NC} $1"; exit 1; }
+
+[ -z "$1" ] && error "Must provide a node version to download"
+NODE_VERSION="${1#v}"; shift # If node version has a v, drop it.
+
+################################################################################
+## Work out version to download
+
+rm -rf node-bin/
+
+DOWNLOAD_DIR="https://nodejs.org/download/release/"
+case $NODE_VERSION in
+  *nightly*) DOWNLOAD_DIR="https://nodejs.org/download/nightly/" ;;
+  *canary*) DOWNLOAD_DIR="https://nodejs.org/download/v8-canary/" ;;
+esac
+
+# Get available versions, grep for NODE_VERSION, and sort by biggest version number.
+nodeGrep="$(echo "$NODE_VERSION" | sed 's/\./\\./g')" # Escape dots.
+availableVersions=$(curl "$DOWNLOAD_DIR" | grep "^<a href="| cut -d \" -f 2 | tr -d /)
+
+if $(echo "$availableVersions" | grep -q "^v\?$nodeGrep$"); then
+  version="$NODE_VERSION"
+else
+  version=$(echo "$availableVersions" | grep "$nodeGrep" |
+    sort -n -t . -k 1.2 -k 2 -k 3 | tail -1)
+fi
+
+# Calculate os and arch.
+os="$(uname | tr '[:upper:]' '[:lower:]')"
+arch=$(uname -m)
+
+case $os in
+  *nt*|*NT*) os=win ext=zip;;
+  aix) arch=ppc64 ;;
+esac
+[ "$os" != win ] && ext=tar.gz
+
+# TODO(gib): Handle x86 SmartOS (currently uses x64).
+# TODO(gib): Handle arm64 machines.
+case $arch in
+  x86_64|i86pc) arch=x64 ;; # i86pc is SmartOS.
+  i686) arch=x86 ;;
+esac
+
+################################################################################
+## Download and extract Node
+
+curl "$DOWNLOAD_DIR$version/node-$version-$os-$arch.$ext" |
+  gzip -cd | tar xf - # Non-GNU tar can't handle gzips.
+mv node-$version-$os-$arch node-bin/

jenkins/scripts/common/helpers.sh

@@ -0,0 +1,36 @@
+# Helper functions and variables. Script should be sourced.
+
+# No POSIX way to get dir of sourced script.
+[ "$BASH_VERSION" ] && thisDir="$(dirname ${BASH_SOURCE[0]})"
+[ "$ZSH_VERSION" ] && thisDir="$(dirname $0)"
+[ -z "$thisDir" ] && { echo "Must be run through bash or ZSH to get sourced dir"; exit 1; }
+
+# Set BUILD_TOOLS to path to BUILD_TOOLS dir, change if you move this file.
+export BUILD_TOOLS="$(cd $thisDir/../../; pwd)"
+
+# Get colour aliases.
+. "$BUILD_TOOLS"/jenkinsnode/common/colours.sh
+
+error() { echo -e "${BRED}ERROR:${NC} $1"; exit 1; }
+warn() { echo -e "${YELLOW}Warning:${NC} $1"; }
+
+# Separator to use in help text.
+sep="${MAGENTA}:${NC}"
+
+
+# Set OS and ARCH variables if unset.
+if [ -z "$OS" -a -z "$ARCH" ]; then
+  OS="$(uname | tr '[:upper:]' '[:lower:]')"
+  ARCH=$(uname -m)
+
+  case $OS in
+    *nt*|*NT*) OS=win ;;
+    aix) ARCH=ppc64 ;;
+  esac
+
+  case $ARCH in
+    x86_64) ARCH=x64 ;;
+    i686) ARCH=x86 ;;
+  esac
+  export OS ARCH
+fi

jenkins/scripts/llnode-continuous-integration.sh

@@ -0,0 +1,40 @@
+#!/bin/bash -ex
+
+[ -z "$NODE_VERSION" ] &&
+[ -z "$LLDB_VERSION" ] &&
+
+rm -rf llnode npm
+
+# Download node tarball.
+bash -$- "$(dirname $0)"/common/getNode.sh ${NODE_VERSION:? Undefined.}
+
+# Download lldb binary
+bash -$- "$(dirname $0)"/common/getLLDB.sh ${LLDB_VERSION:? Undefined.}
+
+export PATH="$PWD/node-bin/bin:$PATH"
+export PATH="$PWD/lldb-bin/usr/bin:$PATH"
+
+arch=$(uname -m)
+export LD_LIBRARY_PATH="$PWD/lldb-bin/usr/lib/${arch}-linux-gnu/"
+
+export npm_config_userconfig="$PWD"/npm/npmrc
+mkdir -p npm/{cache,devdir,tmp}
+
+cat <<!!EOF >npm/npmrc
+tmpdir=$PWD/npm/tmp
+cache=$PWD/npm/cache
+devdir=$PWD/npm/devdir
+loglevel=error
+progress=false
+!!EOF
+
+node -v
+npm -v
+ls
+
+rm -Rf llnode
+git clone --depth=1 -b ${GIT_BRANCH:?Undefined.} https://github.com/${GIT_REPO:?Undefined.}.git llnode
+cd llnode
+
+npm install --nodedir=$(dirname $(dirname $(which node)))/include/node
+npm test