diff --git a/.gitignore b/.gitignore
index 07d38690ff..03cbf93339 100755
--- a/.gitignore
+++ b/.gitignore
@@ -38,6 +38,9 @@ htmlcov
 .coverage
 .tox
 
+# Pytest cache
+.pytest_cache
+
 # pycharm
 .idea
 
diff --git a/Dockerfile b/Dockerfile
index d73f8c8fca..cb71f2f6ff 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -32,9 +32,6 @@ ADD . /auto-sklearn/
 
 # Upgrade pip then install dependencies
 RUN pip3 install --upgrade pip
-RUN pip3 install pytest>=4.6.* pep8 codecov pytest-cov flake8 openml
-RUN cat /auto-sklearn/requirements.txt | xargs -n 1 -L 1 pip3 install
-RUN pip3 install jupyter
 
 # Install
-RUN pip3 install /auto-sklearn/
+RUN pip3 install "/auto-sklearn[test, examples]"
diff --git a/test/test_scripts/test_metadata_generation.py b/test/test_scripts/test_metadata_generation.py
index 25c4855b08..d888e8589c 100644
--- a/test/test_scripts/test_metadata_generation.py
+++ b/test/test_scripts/test_metadata_generation.py
@@ -13,6 +13,17 @@
 
 import unittest
 
+# Abstracted to here to make changing them easier
+# seems to be quite fidely with github actions
+# Recommended to set a minimum of 60
+timeouts = {
+    1: 60,  # create commands
+    2: 180,  # generate metadata
+    3: 60,  # get performance of configurations
+    4: 90,  # Calculate metafeatures
+    5: 60,  # Create aslib files
+}
+
 
 class TestMetadataGeneration(unittest.TestCase):
     def setUp(self):