Tooling for automated detection of malware

Procfile

@@ -2,4 +2,5 @@ release: bin/release
 web: bin/start-web python -m gunicorn.app.wsgiapp -c gunicorn.conf.py warehouse.wsgi:application
 web-uploads: bin/start-web python -m gunicorn.app.wsgiapp -c gunicorn-uploads.conf.py warehouse.wsgi:application
 worker: bin/start-worker celery -A warehouse worker -Q default -l info --max-tasks-per-child 32
+worker-malware: bin/start-worker celery -A warehouse worker -Q malware -l info --max-tasks-per-child 32
 worker-beat: bin/start-worker celery -A warehouse beat -S redbeat.RedBeatScheduler -l info

bin/release

@@ -5,3 +5,6 @@ set -eo pipefail
 
 # Migrate our database to the latest revision.
 python -m warehouse db upgrade head
+
+# Insert/upgrade malware checks.
+python -m warehouse malware sync-checks

dev/environment

@@ -29,6 +29,8 @@ MAIL_BACKEND=warehouse.email.services.SMTPEmailSender host=smtp port=2525 ssl=fa
 
 BREACHED_PASSWORDS=warehouse.accounts.NullPasswordBreachedService
 
+MALWARE_CHECK_BACKEND=warehouse.malware.services.PrinterMalwareCheckService
+
 METRICS_BACKEND=warehouse.metrics.DataDogMetrics host=notdatadog
 
 STATUSPAGE_URL=https://2p66nmmycsj3.statuspage.io

docker-compose.yml

@@ -93,6 +93,7 @@ services:
     env_file: dev/environment
     environment:
       C_FORCE_ROOT: "1"
+      FILES_BACKEND: "warehouse.packaging.services.LocalFileStorage path=/var/opt/warehouse/packages/ url=http://files:9001/packages/{path}"
     links:
       - db
       - redis

requirements/main.in

@@ -55,5 +55,6 @@ typeguard
 webauthn
 whitenoise
 WTForms>=2.0.0
+yara-python
 zope.sqlalchemy
 zxcvbn

requirements/main.txt

@@ -594,6 +594,20 @@ wired==0.2.1 \
 wtforms==2.2.1 \
     --hash=sha256:0cdbac3e7f6878086c334aa25dc5a33869a3954e9d1e015130d65a69309b3b61 \
     --hash=sha256:e3ee092c827582c50877cdbd49e9ce6d2c5c1f6561f849b3b068c1b8029626f1
+yara-python==3.11.0 \
+    --hash=sha256:105d851e050b32951ee577148c7f1b18c0a7c64432fef8159069191d522fba86 \
+    --hash=sha256:1d35c7f606465015de02143dfa4e1ad2f4ee85fdb5d5af756b51b2bac62ac7bc \
+    --hash=sha256:24cd492d6bf8ecedb128f5b02886770be9df03bd1b84ab06a978d45bb1a8ff92 \
+    --hash=sha256:58cfc837e7769811afbfb19b1db952ec01e50cdbf9df576fb587e1e343694526 \
+    --hash=sha256:5b8d708751a66d1507d819218d06baccdf5527c147c2bd3062f087e2f367a17d \
+    --hash=sha256:6f90bb264470235549e1bb4e355fa82895409cd46f27aceecaddfbf55e66ed71 \
+    --hash=sha256:70d39c2238c5854e7cd8f11595317dc4d89417e88035d8acca24bcc58a93150f \
+    --hash=sha256:8d255349d69d833bca604b4215bdf499c87357172512273feb934f6442b8e6b2 \
+    --hash=sha256:8e44f9600607cb1d74a0f26df5d0a1c06ea54f4601206124f47f1bbb58e6a374 \
+    --hash=sha256:9e4fafc327e3a343c545dcf5f173fa8bc712aebffe5f034d205c0bac1f1c5df6 \
+    --hash=sha256:c919ee656139ed46a0056e8a3de179bbc98d42a2be6fb85c95b1e2ec65396b34 \
+    --hash=sha256:e4124414d3cff9a10669569a89f585f81c8114b283ab48b2e756e0347a89de0a \
+    --hash=sha256:f104f0bb21a0867f22e750bb4e05de629ec9f37facc84daf963385a86371b0d9
 zipp==2.1.0 \
     --hash=sha256:ccc94ed0909b58ffe34430ea5451f07bc0c76467d7081619a454bf5c98b89e28 \
     --hash=sha256:feae2f18633c32fc71f2de629bfb3bd3c9325cd4419642b1f1da42ee488d9b98

tests/common/checks/__init__.py

@@ -0,0 +1,14 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .hooked import ExampleHookedCheck  # noqa
+from .scheduled import ExampleScheduledCheck  # noqa

tests/common/checks/hooked.py

@@ -0,0 +1,40 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from warehouse.malware.checks.base import MalwareCheckBase
+from warehouse.malware.errors import FatalCheckException
+from warehouse.malware.models import VerdictClassification, VerdictConfidence
+
+
+class ExampleHookedCheck(MalwareCheckBase):
+
+    version = 1
+    short_description = "An example hook-based check"
+    long_description = "The purpose of this check is to test the \
+implementation of a hook-based check. This check will generate verdicts if enabled."
+    check_type = "event_hook"
+    hooked_object = "File"
+
+    def __init__(self, db):
+        super().__init__(db)
+
+    def scan(self, **kwargs):
+        file_id = kwargs.get("obj_id")
+        if file_id is None:
+            raise FatalCheckException("Missing required kwarg `obj_id`")
+
+        self.add_verdict(
+            file_id=file_id,
+            classification=VerdictClassification.Benign,
+            confidence=VerdictConfidence.High,
+            message="Nothing to see here!",
+        )

tests/common/checks/scheduled.py

@@ -0,0 +1,37 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from warehouse.malware.checks.base import MalwareCheckBase
+from warehouse.malware.models import VerdictClassification, VerdictConfidence
+from warehouse.packaging.models import Project
+
+
+class ExampleScheduledCheck(MalwareCheckBase):
+
+    version = 1
+    short_description = "An example scheduled check"
+    long_description = "The purpose of this check is to test the \
+implementation of a scheduled check. This check will generate verdicts if enabled."
+    check_type = "scheduled"
+    schedule = {"minute": "0", "hour": "*/8"}
+
+    def __init__(self, db):
+        super().__init__(db)
+
+    def scan(self, **kwargs):
+        project = self.db.query(Project).first()
+        self.add_verdict(
+            project_id=project.id,
+            classification=VerdictClassification.Benign,
+            confidence=VerdictConfidence.High,
+            message="Nothing to see here!",
+        )

tests/common/db/malware.py

@@ -0,0 +1,63 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import datetime
+
+import factory
+import factory.fuzzy
+
+from warehouse.malware.models import (
+    MalwareCheck,
+    MalwareCheckObjectType,
+    MalwareCheckState,
+    MalwareCheckType,
+    MalwareVerdict,
+    VerdictClassification,
+    VerdictConfidence,
+)
+
+from .base import WarehouseFactory
+from .packaging import FileFactory
+
+
+class MalwareCheckFactory(WarehouseFactory):
+    class Meta:
+        model = MalwareCheck
+
+    name = factory.fuzzy.FuzzyText(length=12)
+    version = 1
+    short_description = factory.fuzzy.FuzzyText(length=80)
+    long_description = factory.fuzzy.FuzzyText(length=300)
+    check_type = factory.fuzzy.FuzzyChoice(list(MalwareCheckType))
+    hooked_object = factory.fuzzy.FuzzyChoice(list(MalwareCheckObjectType))
+    schedule = {"minute": "*/10"}
+    state = factory.fuzzy.FuzzyChoice(list(MalwareCheckState))
+    created = factory.fuzzy.FuzzyNaiveDateTime(
+        datetime.datetime.utcnow() - datetime.timedelta(days=7)
+    )
+
+
+class MalwareVerdictFactory(WarehouseFactory):
+    class Meta:
+        model = MalwareVerdict
+
+    check = factory.SubFactory(MalwareCheckFactory)
+    release_file = factory.SubFactory(FileFactory)
+    release = None
+    project = None
+    manually_reviewed = True
+    reviewer_verdict = factory.fuzzy.FuzzyChoice(list(VerdictClassification))
+    classification = factory.fuzzy.FuzzyChoice(list(VerdictClassification))
+    confidence = factory.fuzzy.FuzzyChoice(list(VerdictConfidence))
+    message = factory.fuzzy.FuzzyText(length=80)
+    full_report_link = None
+    details = None

tests/common/db/packaging.py

@@ -83,6 +83,7 @@ class Meta:
 
     release = factory.SubFactory(ReleaseFactory)
     python_version = "source"
+    filename = factory.fuzzy.FuzzyText(length=12)
     md5_digest = factory.LazyAttribute(
         lambda o: hashlib.md5(o.filename.encode("utf8")).hexdigest()
     )

tests/conftest.py

@@ -174,6 +174,9 @@ def app_config(database):
         "files.backend": "warehouse.packaging.services.LocalFileStorage",
         "docs.backend": "warehouse.packaging.services.LocalFileStorage",
         "mail.backend": "warehouse.email.services.SMTPEmailSender",
+        "malware_check.backend": (
+            "warehouse.malware.services.PrinterMalwareCheckService"
+        ),
         "files.url": "http://localhost:7000/",
         "sessions.secret": "123456",
         "sessions.url": "redis://localhost:0/",

tests/unit/admin/test_routes.py

@@ -123,4 +123,27 @@ def test_includeme():
         pretend.call("admin.flags.edit", "/admin/flags/edit/", domain=warehouse),
         pretend.call("admin.squats", "/admin/squats/", domain=warehouse),
         pretend.call("admin.squats.review", "/admin/squats/review/", domain=warehouse),
+        pretend.call("admin.checks.list", "/admin/checks/", domain=warehouse),
+        pretend.call(
+            "admin.checks.detail", "/admin/checks/{check_name}", domain=warehouse
+        ),
+        pretend.call(
+            "admin.checks.change_state",
+            "/admin/checks/{check_name}/change_state",
+            domain=warehouse,
+        ),
+        pretend.call(
+            "admin.checks.run_evaluation",
+            "/admin/checks/{check_name}/run_evaluation",
+            domain=warehouse,
+        ),
+        pretend.call("admin.verdicts.list", "/admin/verdicts/", domain=warehouse),
+        pretend.call(
+            "admin.verdicts.detail", "/admin/verdicts/{verdict_id}", domain=warehouse
+        ),
+        pretend.call(
+            "admin.verdicts.review",
+            "/admin/verdicts/{verdict_id}/review",
+            domain=warehouse,
+        ),
     ]