Add M1 macs (#2565)

* ansible: add support for apple sillicon

* ansible: add M1 Macs

Also clean up some out of date machines

* ansible: Add new m1 macs

* clean up inventory file

* update broken steps

* fixup! update broken steps

* ansible: change archtype in jenkins start script

* fixup! ansible: change archtype in jenkins start script

* add missing slash

* fixup some script errors

* ansible: remove DTKs

Author: Ash Cripps

ansible/inventory.yml

@@ -51,8 +51,13 @@ hosts:
         ubuntu1804_docker-x64-1: {ip: 147.75.88.56, user: ubuntu}
 
     - macstadium:
-        macos10.11-x64-1: {ip: 207.254.58.162, port: 10013, user: administrator}
-        macos10.10-x64-1: {ip: 207.254.58.162, port: 10014, user: administrator}
+        macos11.0-arm64-1:
+            ansible_python_interpreter: /usr/bin/python3
+            ip: 207.254.38.74
+            user: administrator
+            remote_env:
+                PATH: /opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin
+            server_jobs: 6
 
     - marist:
         zos24-s390x-1: {ip: 148.100.36.157, user: unix1}
@@ -151,8 +156,18 @@ hosts:
         ubuntu1804-x64-1: {ip: 147.75.88.51, user: ubuntu}
 
     - macstadium:
-        macos11.0-arm64-1: {ip: 199.7.163.9, user: administrator}
-        macos11.0-arm64-2: {ip: 199.7.163.10, user: administrator}
+        macos11.0-arm64-3:
+            ansible_python_interpreter: /usr/bin/python3
+            ip: 207.254.38.86
+            user: administrator
+            remote_env:
+                PATH: /opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin
+        macos11.0-arm64-4:
+            ansible_python_interpreter: /usr/bin/python3
+            ip: 207.254.38.89
+            user: administrator
+            remote_env:
+                PATH: /opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin
 
     - marist:
         zos24-s390x-1:
@@ -185,14 +200,6 @@ hosts:
         ubuntu1604-arm64_odroid_c2-2: {ip: 70.167.220.148}
         ubuntu1604-arm64_odroid_c2-3: {ip: 70.167.220.149, user: odroid}
 
-    - macstadium:
-        macos10.10-x64-1: {ip: 207.254.58.162, port: 10005, user: administrator}
-        macos10.10-x64-2: {ip: 207.254.58.162, port: 10006, user: administrator}
-        macos10.11-x64-1: {ip: 207.254.58.162, port: 10003, user: administrator}
-        macos10.11-x64-2: {ip: 207.254.58.162, port: 10004, user: administrator}
-        macos10.12-x64-1: {ip: 207.254.58.162, port: 10001, user: administrator}
-        macos10.12-x64-2: {ip: 207.254.58.162, port: 10002, user: administrator}
-
     - msft:
         win10_vs2017-arm64-1: {}
         win10_vs2017-arm64-2: {}

ansible/playbooks/jenkins/worker/create.yml

@@ -84,7 +84,7 @@
   - "!test-ibm-ubuntu1804-x64-1"
   tasks:
     - name: remove node and npm packages
-      when: not os|startswith("win") and not os|startswith("zos") and not os|startswith("ibmi")
+      when: not os|startswith("win") and not os|startswith("zos") and not os|startswith("ibmi") and os != "macos11.0"
       package:
         name: "{{ package }}"
         state: absent

ansible/roles/baselayout/tasks/main.yml

@@ -86,7 +86,7 @@
 - name: install packages (macos)
   when: os|startswith("macos")
   become_user: administrator
-  package: name="{{ package }}" state=present
+  community.general.homebrew: name="{{ package }}" state=present
   loop_control:
     loop_var: package
   with_items:

ansible/roles/baselayout/tasks/partials/ccache/macos.yml

@@ -8,6 +8,6 @@
 - name: "ccache : add ccache to the path (macos)"
   when: ccache_mac.rc == 1
   lineinfile:
-    dest: "etc/paths"
+    dest: "/etc/paths"
     insertbefore: BOF
-    line: "usr/local/opt/ccache/libexec"
+    line: "/usr/local/opt/ccache/libexec"

ansible/roles/java-base/tasks/main.yml

@@ -53,18 +53,30 @@
 
 - name: install java tap (macOS)
   become_user: administrator
-  when: java.rc > 0 and os|startswith("macos")
+  when: java.rc > 0 and os|startswith("macos10")
   homebrew_tap:
         name: AdoptOpenJDK/openjdk
         state: present
 
 - name: install java (macOS)
   become_user: administrator
-  when: java.rc > 0 and os|startswith("macos")
+  when: java.rc > 0 and os|startswith("macos10")
   homebrew_cask:
         name: "{{ java_package_name }}"
         state: present
 
+- name: Fetch java (Apple Silicon)
+  when: os|startswith("macos11")
+  shell:
+    chdir: "/Users/administrator"
+    cmd: "curl -L -o zulu8.52.0.23-ca-jdk8.0.282-macosx_aarch64.tar.gz https://cdn.azul.com/zulu/bin/zulu8.52.0.23-ca-jdk8.0.282-macosx_aarch64.tar.gz"
+
+- name: Extract java (Apple Silicon)
+  when: os|startswith("macos11")
+  shell:
+    chdir: "/Users/administrator"
+    cmd: "tar -xf zulu8.52.0.23-ca-jdk8.0.282-macosx_aarch64.tar.gz"
+
 - name: install webupd8 oracle java 8 extras
   when: java.rc > 0 and os == "ubuntu1404" and arch != "ppc64"
   package: name="{{item}}" state=present
@@ -151,4 +163,4 @@
     - use_adoptopenjdk == True
     - update_adoptopenjdk == True
     - adoptopenjdk_symlink.stat.exists
-    - adoptopenjdk_symlink.stat.lnk_source != "/opt/"+adoptopenjdk_unpacked.files[0].split('/')[0]+"/bin/java"
+    - adoptopenjdk_symlink.stat.lnk_source != "/opt/"+adoptopenjdk_unpacked.files[0].split('/')[0]+"/bin/java"

ansible/roles/java-base/vars/main.yml

@@ -13,7 +13,7 @@ packages: {
   'fedora': 'java-1.8.0-openjdk-headless',
   'freebsd': 'openjdk8-jre',
   'ibmi': 'openjdk-11-ea',
-  'macos': 'adoptopenjdk8',
+  'macos10': 'adoptopenjdk8',
   'rhel7': 'java-1.8.0-openjdk',
   'smartos': 'openjdk8',
   'ubuntu': 'openjdk-8-jre-headless',
@@ -36,4 +36,4 @@ adoptopenjdk: {
 adoptopenjdk_arch: "{{ adoptopenjdk[os+'_'+arch].arch | default(ansible_architecture) }}"
 adoptopenjdk_os: "{{ adoptopenjdk[os+'_'+arch].os | default(ansible_system | lower) }}"
 adoptopenjdk_version: "{{ adoptopenjdk[os+'_'+arch].version | default('8') }}"
-use_adoptopenjdk: "{{ adoptopenjdk[os+'_'+arch] is defined | bool }}"
+use_adoptopenjdk: "{{ adoptopenjdk[os+'_'+arch] is defined | bool }}"

ansible/roles/jenkins-worker/templates/start.j2

@@ -4,9 +4,9 @@ export NODE_TEST_DIR="$HOME/tmp"
 export JOBS="{{ jobs_env }}"
 
 export OSTYPE=osx
-export ARCH=x64
-export DESTCPU=x64
+export ARCH="{{ arch }}"
+export DESTCPU="{{ arch }}"
 
-PATH="/usr/local/opt/ccache/libexec:/usr/local/opt/python3/Frameworks/Python.framework/Versions/Current/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin" {{ java_path[os] }} -Xmx{{ server_ram|default('128m') }} \
+PATH="/usr/local/opt/ccache/libexec:/usr/local/opt/python3/Frameworks/Python.framework/Versions/Current/bin:/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin" {{ java_path[os] }} -Xmx{{ server_ram|default('128m') }} \
     -jar {{ home }}/{{ server_user }}/slave.jar -secret {{ secret }} \
     -jnlpUrl {{ jenkins_url }}/computer/{{ inventory_hostname }}/slave-agent.jnlp

ansible/roles/jenkins-worker/vars/main.yml

@@ -97,7 +97,8 @@ java_path: {
   'macos10.14': 'java',
   'macos10.15': 'java',
   'macos10.16': 'java',
-  'macos11.0': 'java',
+  # Currently hardcoded untill adopt have their build available
+  'macos11.0': '/Users/administrator/zulu8.52.0.23-ca-jdk8.0.282-macosx_aarch64/bin/java',
   'smartos15': '/opt/local/java/openjdk8/bin/java',
   'smartos16': '/opt/local/java/openjdk8/bin/java',
   'smartos17': '/opt/local/java/openjdk8/bin/java',

ansible/roles/package-upgrade/tasks/partials/brew.yml

@@ -13,15 +13,27 @@
     script: files/install-xcode.sh
     when: xcode.rc > 1
 
+  - name: Check if Homebrew is already installed (Apple Sillicon)
+    stat:
+      path: /opt/homebrew/bin/brew
+    register: armbrew
+    when: os == "macos11.0"
+
   - name: Check if Homebrew is already installed
     stat:
       path: /usr/local/bin/brew
     register: brew
+    when: os != "macos11.0"
+
+  - name: Install Homebrew
+    become_user: administrator
+    script: files/install-homebrew.sh
+    when: os != "macos11.0" and not brew.stat.exists
 
   - name: Install Homebrew
     become_user: administrator
     script: files/install-homebrew.sh
-    when: not brew.stat.exists
+    when: os == "macos11.0" and not armbrew.stat.exists
 
   - name: Upgrade installed packages
     become_user: administrator