Skip to content

Commit d4dd8f5

Browse files
ewjoachimewjoachim
committed
Add notgithub service to simulate github token scanning
1 parent fa2c5f9 commit d4dd8f5

File tree

8 files changed

+93
-23
lines changed

8 files changed

+93
-23
lines changed

‎dev/environment

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,3 +43,5 @@ WAREHOUSE_LEGACY_DOMAIN=pypi.python.org
4343

4444
VAULT_URL="http://vault:8200"
4545
VAULT_TOKEN="an insecure vault access token"
46+
47+
GITHUB_TOKEN_SCANNING_META_API_URL="http://notgithub:8000/meta/public_keys/token_scanning"

‎docker-compose.yml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,3 +142,10 @@ services:
142142
- "8125:8125/udp"
143143
volumes:
144144
- ./dev/notdatadog.py:/opt/warehouse/dev/notdatadog.py
145+
146+
notgithub:
147+
image: ewjoachim/notgithub-token-scanning
148+
environment:
149+
NOTGITHUB_DEFAULT_URL: "http://web:8000/_/github/disclose-token"
150+
ports:
151+
- "8964:8000"

‎tests/unit/integration/github/test_utils.py

Lines changed: 47 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -113,15 +113,21 @@ def test_init(self):
113113
metrics = pretend.stub()
114114
session = pretend.stub()
115115
token = "api_token"
116+
url = "http://foo"
116117
cache = utils.PublicKeysCache(cache_time=12)
117118

118119
verifier = utils.GitHubTokenScanningPayloadVerifier(
119-
session=session, metrics=metrics, api_token=token, public_keys_cache=cache
120+
api_url=url,
121+
session=session,
122+
metrics=metrics,
123+
api_token=token,
124+
public_keys_cache=cache,
120125
)
121126

122127
assert verifier._session is session
123128
assert verifier._metrics is metrics
124129
assert verifier._api_token == token
130+
assert verifier._api_url == url
125131
assert verifier._public_keys_cache is cache
126132

127133
def test_verify_cache_miss(self):
@@ -147,6 +153,7 @@ def test_verify_cache_miss(self):
147153
metrics = pretend.stub(increment=pretend.call_recorder(lambda str: None))
148154
cache = utils.PublicKeysCache(cache_time=12)
149155
verifier = utils.GitHubTokenScanningPayloadVerifier(
156+
api_url="http://foo",
150157
session=session,
151158
metrics=metrics,
152159
api_token="api-token",
@@ -188,6 +195,7 @@ def test_verify_cache_hit(self):
188195
}
189196
]
190197
verifier = utils.GitHubTokenScanningPayloadVerifier(
198+
api_url="http://foo",
191199
session=session,
192200
metrics=metrics,
193201
api_token="api-token",
@@ -218,6 +226,7 @@ def test_verify_error(self):
218226
metrics = pretend.stub(increment=pretend.call_recorder(lambda str: None))
219227
cache = utils.PublicKeysCache(cache_time=12)
220228
verifier = utils.GitHubTokenScanningPayloadVerifier(
229+
api_url="http://foo",
221230
session=pretend.stub(),
222231
metrics=metrics,
223232
api_token="api-token",
@@ -236,6 +245,7 @@ def test_verify_error(self):
236245

237246
def test_headers_auth_no_token(self):
238247
headers = utils.GitHubTokenScanningPayloadVerifier(
248+
api_url="http://foo",
239249
session=pretend.stub(),
240250
metrics=pretend.stub(),
241251
api_token=None,
@@ -245,6 +255,7 @@ def test_headers_auth_no_token(self):
245255

246256
def test_headers_auth_token(self):
247257
headers = utils.GitHubTokenScanningPayloadVerifier(
258+
api_url="http://foo",
248259
session=pretend.stub(),
249260
metrics=pretend.stub(),
250261
api_token="api-token",
@@ -273,6 +284,7 @@ def test_retrieve_public_key_payload(self):
273284
metrics = pretend.stub(increment=pretend.call_recorder(lambda str: None))
274285

275286
verifier = utils.GitHubTokenScanningPayloadVerifier(
287+
api_url="http://foo",
276288
session=session,
277289
metrics=metrics,
278290
api_token="api-token",
@@ -281,7 +293,7 @@ def test_retrieve_public_key_payload(self):
281293
assert verifier._retrieve_public_key_payload() == meta_payload
282294
assert session.get.calls == [
283295
pretend.call(
284-
"https://api.github.com/meta/public_keys/token_scanning",
296+
"http://foo",
285297
headers={"Authorization": "token api-token"},
286298
)
287299
]
@@ -294,7 +306,10 @@ def test_get_cached_public_key_cache_hit(self):
294306
cache.set(now=time.time(), value=cache_value)
295307

296308
verifier = utils.GitHubTokenScanningPayloadVerifier(
297-
session=session, metrics=metrics, public_keys_cache=cache
309+
api_url="http://foo",
310+
session=session,
311+
metrics=metrics,
312+
public_keys_cache=cache,
298313
)
299314

300315
assert verifier._get_cached_public_keys() is cache_value
@@ -305,7 +320,10 @@ def test_get_cached_public_key_cache_miss_no_cache(self):
305320
cache = utils.PublicKeysCache(cache_time=12)
306321

307322
verifier = utils.GitHubTokenScanningPayloadVerifier(
308-
session=session, metrics=metrics, public_keys_cache=cache
323+
api_url="http://foo",
324+
session=session,
325+
metrics=metrics,
326+
public_keys_cache=cache,
309327
)
310328

311329
with pytest.raises(utils.CacheMiss):
@@ -321,7 +339,10 @@ def test_retrieve_public_key_payload_http_error(self):
321339
get=lambda *a, **k: response,
322340
)
323341
verifier = utils.GitHubTokenScanningPayloadVerifier(
324-
session=session, metrics=pretend.stub(), public_keys_cache=pretend.stub()
342+
api_url="http://foo",
343+
session=session,
344+
metrics=pretend.stub(),
345+
public_keys_cache=pretend.stub(),
325346
)
326347
with pytest.raises(utils.GitHubPublicKeyMetaAPIError) as exc:
327348
verifier._retrieve_public_key_payload()
@@ -337,7 +358,10 @@ def test_retrieve_public_key_payload_json_error(self):
337358
)
338359
session = pretend.stub(get=lambda *a, **k: response)
339360
verifier = utils.GitHubTokenScanningPayloadVerifier(
340-
session=session, metrics=pretend.stub(), public_keys_cache=pretend.stub()
361+
api_url="http://foo",
362+
session=session,
363+
metrics=pretend.stub(),
364+
public_keys_cache=pretend.stub(),
341365
)
342366
with pytest.raises(utils.GitHubPublicKeyMetaAPIError) as exc:
343367
verifier._retrieve_public_key_payload()
@@ -349,7 +373,10 @@ def test_retrieve_public_key_payload_connection_error(self):
349373
session = pretend.stub(get=pretend.raiser(requests.ConnectionError))
350374

351375
verifier = utils.GitHubTokenScanningPayloadVerifier(
352-
session=session, metrics=pretend.stub(), public_keys_cache=pretend.stub()
376+
api_url="http://foo",
377+
session=session,
378+
metrics=pretend.stub(),
379+
public_keys_cache=pretend.stub(),
353380
)
354381

355382
with pytest.raises(utils.GitHubPublicKeyMetaAPIError) as exc:
@@ -374,7 +401,10 @@ def test_extract_public_keys(self):
374401
}
375402
cache = utils.PublicKeysCache(cache_time=12)
376403
verifier = utils.GitHubTokenScanningPayloadVerifier(
377-
session=pretend.stub(), metrics=pretend.stub(), public_keys_cache=cache
404+
api_url="http://foo",
405+
session=pretend.stub(),
406+
metrics=pretend.stub(),
407+
public_keys_cache=cache,
378408
)
379409

380410
keys = verifier._extract_public_keys(pubkey_api_data=meta_payload)
@@ -414,7 +444,10 @@ def test_extract_public_keys(self):
414444
def test_extract_public_keys_error(self, payload, expected):
415445
cache = utils.PublicKeysCache(cache_time=12)
416446
verifier = utils.GitHubTokenScanningPayloadVerifier(
417-
session=pretend.stub(), metrics=pretend.stub(), public_keys_cache=cache
447+
api_url="http://foo",
448+
session=pretend.stub(),
449+
metrics=pretend.stub(),
450+
public_keys_cache=cache,
418451
)
419452

420453
with pytest.raises(utils.GitHubPublicKeyMetaAPIError) as exc:
@@ -426,6 +459,7 @@ def test_extract_public_keys_error(self, payload, expected):
426459

427460
def test_check_public_key(self):
428461
verifier = utils.GitHubTokenScanningPayloadVerifier(
462+
api_url="http://foo",
429463
session=pretend.stub(),
430464
metrics=pretend.stub(),
431465
public_keys_cache=pretend.stub(),
@@ -439,6 +473,7 @@ def test_check_public_key(self):
439473

440474
def test_check_public_key_error(self):
441475
verifier = utils.GitHubTokenScanningPayloadVerifier(
476+
api_url="http://foo",
442477
session=pretend.stub(),
443478
metrics=pretend.stub(),
444479
public_keys_cache=pretend.stub(),
@@ -452,6 +487,7 @@ def test_check_public_key_error(self):
452487

453488
def test_check_signature(self):
454489
verifier = utils.GitHubTokenScanningPayloadVerifier(
490+
api_url="http://foo",
455491
session=pretend.stub(),
456492
metrics=pretend.stub(),
457493
public_keys_cache=pretend.stub(),
@@ -481,6 +517,7 @@ def test_check_signature(self):
481517

482518
def test_check_signature_invalid_signature(self):
483519
verifier = utils.GitHubTokenScanningPayloadVerifier(
520+
api_url="http://foo",
484521
session=pretend.stub(),
485522
metrics=pretend.stub(),
486523
public_keys_cache=pretend.stub(),
@@ -512,6 +549,7 @@ def test_check_signature_invalid_signature(self):
512549

513550
def test_check_signature_invalid_crypto(self):
514551
verifier = utils.GitHubTokenScanningPayloadVerifier(
552+
api_url="http://foo",
515553
session=pretend.stub(),
516554
metrics=pretend.stub(),
517555
public_keys_cache=pretend.stub(),

‎tests/unit/integration/github/test_views.py

Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,10 @@ def test_github_disclose_token(self, pyramid_request, monkeypatch):
2929

3030
pyramid_request.body = "[1, 2, 3]"
3131
pyramid_request.json_body = [1, 2, 3]
32-
pyramid_request.registry.settings = {"github.token": "token"}
32+
pyramid_request.registry.settings = {
33+
"github.token": "token",
34+
"github.token_scanning_meta_api.url": "http://foo",
35+
}
3336
pyramid_request.find_service = lambda *a, **k: metrics
3437

3538
http = pyramid_request.http = pretend.stub()
@@ -46,7 +49,9 @@ def test_github_disclose_token(self, pyramid_request, monkeypatch):
4649

4750
assert response.status_code == 204
4851
assert verifier_cls.calls == [
49-
pretend.call(session=http, metrics=metrics, api_token="token")
52+
pretend.call(
53+
session=http, metrics=metrics, api_token="token", api_url="http://foo"
54+
)
5055
]
5156
assert verify.calls == [
5257
pretend.call(payload="[1, 2, 3]", key_id="foo", signature="bar")
@@ -70,7 +75,9 @@ def test_github_disclose_token_no_token(self, pyramid_request, monkeypatch):
7075

7176
pyramid_request.body = "[1, 2, 3]"
7277
pyramid_request.json_body = [1, 2, 3]
73-
pyramid_request.registry.settings = {}
78+
pyramid_request.registry.settings = {
79+
"github.token_scanning_meta_api.url": "http://foo"
80+
}
7481
pyramid_request.find_service = lambda *a, **k: metrics
7582
pyramid_request.http = pretend.stub()
7683

@@ -96,7 +103,10 @@ def test_github_disclose_token_verify_fail(self, monkeypatch, pyramid_request):
96103

97104
pyramid_request.body = "[1, 2, 3]"
98105
pyramid_request.find_service = lambda *a, **k: metrics
99-
pyramid_request.registry.settings = {"github.token": "token"}
106+
pyramid_request.registry.settings = {
107+
"github.token": "token",
108+
"github.token_scanning_meta_api.url": "http://foo",
109+
}
100110

101111
pyramid_request.http = pretend.stub()
102112

@@ -137,7 +147,12 @@ def find_service(self, *a, **k):
137147

138148
response = pretend.stub(status_int=200)
139149
http = pretend.stub()
140-
registry = pretend.stub(settings={"github.token": "token"})
150+
registry = pretend.stub(
151+
settings={
152+
"github.token": "token",
153+
"github.token_scanning_meta_api.url": "http://foo",
154+
}
155+
)
141156

142157
request = Request()
143158
response = views.github_disclose_token(request)
@@ -160,7 +175,10 @@ def metrics_increment(key):
160175

161176
pyramid_request.body = "{}"
162177
pyramid_request.json_body = {}
163-
pyramid_request.registry.settings = {"github.token": "token"}
178+
pyramid_request.registry.settings = {
179+
"github.token": "token",
180+
"github.token_scanning_meta_api.url": "http://foo",
181+
}
164182
pyramid_request.find_service = lambda *a, **k: metrics_service
165183

166184
pyramid_request.http = pretend.stub()

‎tests/unit/test_config.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -231,6 +231,9 @@ def __init__(self):
231231
"token.default.max_age": 21600,
232232
"warehouse.xmlrpc.client.ratelimit_string": "3600 per hour",
233233
"warehouse.xmlrpc.search.enabled": True,
234+
"github.token_scanning_meta_api.url": (
235+
"https://github.com/api/meta/public_keys/token_scanning"
236+
),
234237
}
235238
if environment == config.Environment.development:
236239
expected_settings.update(

‎warehouse/config.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,12 @@ def configure(settings=None):
164164
settings, "warehouse.release_files_table", "WAREHOUSE_RELEASE_FILES_TABLE"
165165
)
166166
maybe_set(settings, "github.token", "GITHUB_TOKEN")
167+
maybe_set(
168+
settings,
169+
"github.token_scanning_meta_api.url",
170+
"GITHUB_TOKEN_SCANNING_META_API_URL",
171+
default="https://github.com/api/meta/public_keys/token_scanning",
172+
)
167173
maybe_set(settings, "warehouse.trending_table", "WAREHOUSE_TRENDING_TABLE")
168174
maybe_set(settings, "celery.broker_url", "BROKER_URL")
169175
maybe_set(settings, "celery.result_url", "REDIS_URL")

‎warehouse/integrations/github/utils.py

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -164,13 +164,15 @@ def __init__(
164164
*,
165165
session,
166166
metrics,
167+
api_url: str,
167168
api_token: Optional[str] = None,
168169
public_keys_cache=PUBLIC_KEYS_CACHE,
169170
):
170171
self._metrics = metrics
171172
self._session = session
172173
self._api_token = api_token
173174
self._public_keys_cache = public_keys_cache
175+
self._api_url = api_url
174176

175177
def verify(self, *, payload, key_id, signature):
176178

@@ -217,15 +219,8 @@ def _headers_auth(self):
217219
return {"Authorization": f"token {self._api_token}"}
218220

219221
def _retrieve_public_key_payload(self):
220-
221-
token_scanning_pubkey_api_url = (
222-
"https://github.com/api/meta/public_keys/token_scanning"
223-
)
224-
225222
try:
226-
response = self._session.get(
227-
token_scanning_pubkey_api_url, headers=self._headers_auth()
228-
)
223+
response = self._session.get(self._api_url, headers=self._headers_auth())
229224
response.raise_for_status()
230225
return response.json()
231226
except requests.HTTPError as exc:

‎warehouse/integrations/github/views.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ def github_disclose_token(request):
4747
verifier = utils.GitHubTokenScanningPayloadVerifier(
4848
session=request.http,
4949
metrics=metrics,
50+
api_url=request.registry.settings["github.token_scanning_meta_api.url"],
5051
api_token=request.registry.settings.get("github.token"),
5152
)
5253

0 commit comments

Comments
 (0)