Skip to content

Commit 6389d73

Browse files
ewjoachimewjoachim
committed
Add notgithub service to simulate github token scanning
1 parent 6f784fc commit 6389d73

File tree

8 files changed

+93
-23
lines changed

8 files changed

+93
-23
lines changed

‎dev/environment

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,3 +43,5 @@ WAREHOUSE_LEGACY_DOMAIN=pypi.python.org
4343

4444
VAULT_URL="http://vault:8200"
4545
VAULT_TOKEN="an insecure vault access token"
46+
47+
GITHUB_TOKEN_SCANNING_META_API_URL="http://notgithub:8000/meta/public_keys/token_scanning"

‎docker-compose.yml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,3 +142,10 @@ services:
142142
- "8125:8125/udp"
143143
volumes:
144144
- ./dev/notdatadog.py:/opt/warehouse/dev/notdatadog.py
145+
146+
notgithub:
147+
image: ewjoachim/notgithub-token-scanning
148+
environment:
149+
NOTGITHUB_DEFAULT_URL: "http://web:8000/_/github/disclose-token"
150+
ports:
151+
- "8964:8000"

‎tests/unit/integration/github/test_utils.py

Lines changed: 47 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -114,15 +114,21 @@ def test_init(self):
114114
metrics = pretend.stub()
115115
session = pretend.stub()
116116
token = "api_token"
117+
url = "http://foo"
117118
cache = utils.PublicKeysCache(cache_time=12)
118119

119120
verifier = utils.GitHubTokenScanningPayloadVerifier(
120-
session=session, metrics=metrics, api_token=token, public_keys_cache=cache
121+
api_url=url,
122+
session=session,
123+
metrics=metrics,
124+
api_token=token,
125+
public_keys_cache=cache,
121126
)
122127

123128
assert verifier._session is session
124129
assert verifier._metrics is metrics
125130
assert verifier._api_token == token
131+
assert verifier._api_url == url
126132
assert verifier._public_keys_cache is cache
127133

128134
def test_verify_cache_miss(self):
@@ -148,6 +154,7 @@ def test_verify_cache_miss(self):
148154
metrics = pretend.stub(increment=pretend.call_recorder(lambda str: None))
149155
cache = utils.PublicKeysCache(cache_time=12)
150156
verifier = utils.GitHubTokenScanningPayloadVerifier(
157+
api_url="http://foo",
151158
session=session,
152159
metrics=metrics,
153160
api_token="api-token",
@@ -189,6 +196,7 @@ def test_verify_cache_hit(self):
189196
}
190197
]
191198
verifier = utils.GitHubTokenScanningPayloadVerifier(
199+
api_url="http://foo",
192200
session=session,
193201
metrics=metrics,
194202
api_token="api-token",
@@ -219,6 +227,7 @@ def test_verify_error(self):
219227
metrics = pretend.stub(increment=pretend.call_recorder(lambda str: None))
220228
cache = utils.PublicKeysCache(cache_time=12)
221229
verifier = utils.GitHubTokenScanningPayloadVerifier(
230+
api_url="http://foo",
222231
session=pretend.stub(),
223232
metrics=metrics,
224233
api_token="api-token",
@@ -237,6 +246,7 @@ def test_verify_error(self):
237246

238247
def test_headers_auth_no_token(self):
239248
headers = utils.GitHubTokenScanningPayloadVerifier(
249+
api_url="http://foo",
240250
session=pretend.stub(),
241251
metrics=pretend.stub(),
242252
api_token=None,
@@ -246,6 +256,7 @@ def test_headers_auth_no_token(self):
246256

247257
def test_headers_auth_token(self):
248258
headers = utils.GitHubTokenScanningPayloadVerifier(
259+
api_url="http://foo",
249260
session=pretend.stub(),
250261
metrics=pretend.stub(),
251262
api_token="api-token",
@@ -274,6 +285,7 @@ def test_retrieve_public_key_payload(self):
274285
metrics = pretend.stub(increment=pretend.call_recorder(lambda str: None))
275286

276287
verifier = utils.GitHubTokenScanningPayloadVerifier(
288+
api_url="http://foo",
277289
session=session,
278290
metrics=metrics,
279291
api_token="api-token",
@@ -282,7 +294,7 @@ def test_retrieve_public_key_payload(self):
282294
assert verifier._retrieve_public_key_payload() == meta_payload
283295
assert session.get.calls == [
284296
pretend.call(
285-
"https://api.github.com/meta/public_keys/token_scanning",
297+
"http://foo",
286298
headers={"Authorization": "token api-token"},
287299
)
288300
]
@@ -295,7 +307,10 @@ def test_get_cached_public_key_cache_hit(self):
295307
cache.set(now=time.time(), value=cache_value)
296308

297309
verifier = utils.GitHubTokenScanningPayloadVerifier(
298-
session=session, metrics=metrics, public_keys_cache=cache
310+
api_url="http://foo",
311+
session=session,
312+
metrics=metrics,
313+
public_keys_cache=cache,
299314
)
300315

301316
assert verifier._get_cached_public_keys() is cache_value
@@ -306,7 +321,10 @@ def test_get_cached_public_key_cache_miss_no_cache(self):
306321
cache = utils.PublicKeysCache(cache_time=12)
307322

308323
verifier = utils.GitHubTokenScanningPayloadVerifier(
309-
session=session, metrics=metrics, public_keys_cache=cache
324+
api_url="http://foo",
325+
session=session,
326+
metrics=metrics,
327+
public_keys_cache=cache,
310328
)
311329

312330
with pytest.raises(utils.CacheMiss):
@@ -322,7 +340,10 @@ def test_retrieve_public_key_payload_http_error(self):
322340
get=lambda *a, **k: response,
323341
)
324342
verifier = utils.GitHubTokenScanningPayloadVerifier(
325-
session=session, metrics=pretend.stub(), public_keys_cache=pretend.stub()
343+
api_url="http://foo",
344+
session=session,
345+
metrics=pretend.stub(),
346+
public_keys_cache=pretend.stub(),
326347
)
327348
with pytest.raises(utils.GitHubPublicKeyMetaAPIError) as exc:
328349
verifier._retrieve_public_key_payload()
@@ -338,7 +359,10 @@ def test_retrieve_public_key_payload_json_error(self):
338359
)
339360
session = pretend.stub(get=lambda *a, **k: response)
340361
verifier = utils.GitHubTokenScanningPayloadVerifier(
341-
session=session, metrics=pretend.stub(), public_keys_cache=pretend.stub()
362+
api_url="http://foo",
363+
session=session,
364+
metrics=pretend.stub(),
365+
public_keys_cache=pretend.stub(),
342366
)
343367
with pytest.raises(utils.GitHubPublicKeyMetaAPIError) as exc:
344368
verifier._retrieve_public_key_payload()
@@ -350,7 +374,10 @@ def test_retrieve_public_key_payload_connection_error(self):
350374
session = pretend.stub(get=pretend.raiser(requests.ConnectionError))
351375

352376
verifier = utils.GitHubTokenScanningPayloadVerifier(
353-
session=session, metrics=pretend.stub(), public_keys_cache=pretend.stub()
377+
api_url="http://foo",
378+
session=session,
379+
metrics=pretend.stub(),
380+
public_keys_cache=pretend.stub(),
354381
)
355382

356383
with pytest.raises(utils.GitHubPublicKeyMetaAPIError) as exc:
@@ -375,7 +402,10 @@ def test_extract_public_keys(self):
375402
}
376403
cache = utils.PublicKeysCache(cache_time=12)
377404
verifier = utils.GitHubTokenScanningPayloadVerifier(
378-
session=pretend.stub(), metrics=pretend.stub(), public_keys_cache=cache
405+
api_url="http://foo",
406+
session=pretend.stub(),
407+
metrics=pretend.stub(),
408+
public_keys_cache=cache,
379409
)
380410

381411
keys = verifier._extract_public_keys(pubkey_api_data=meta_payload)
@@ -415,7 +445,10 @@ def test_extract_public_keys(self):
415445
def test_extract_public_keys_error(self, payload, expected):
416446
cache = utils.PublicKeysCache(cache_time=12)
417447
verifier = utils.GitHubTokenScanningPayloadVerifier(
418-
session=pretend.stub(), metrics=pretend.stub(), public_keys_cache=cache
448+
api_url="http://foo",
449+
session=pretend.stub(),
450+
metrics=pretend.stub(),
451+
public_keys_cache=cache,
419452
)
420453

421454
with pytest.raises(utils.GitHubPublicKeyMetaAPIError) as exc:
@@ -427,6 +460,7 @@ def test_extract_public_keys_error(self, payload, expected):
427460

428461
def test_check_public_key(self):
429462
verifier = utils.GitHubTokenScanningPayloadVerifier(
463+
api_url="http://foo",
430464
session=pretend.stub(),
431465
metrics=pretend.stub(),
432466
public_keys_cache=pretend.stub(),
@@ -440,6 +474,7 @@ def test_check_public_key(self):
440474

441475
def test_check_public_key_error(self):
442476
verifier = utils.GitHubTokenScanningPayloadVerifier(
477+
api_url="http://foo",
443478
session=pretend.stub(),
444479
metrics=pretend.stub(),
445480
public_keys_cache=pretend.stub(),
@@ -453,6 +488,7 @@ def test_check_public_key_error(self):
453488

454489
def test_check_signature(self):
455490
verifier = utils.GitHubTokenScanningPayloadVerifier(
491+
api_url="http://foo",
456492
session=pretend.stub(),
457493
metrics=pretend.stub(),
458494
public_keys_cache=pretend.stub(),
@@ -482,6 +518,7 @@ def test_check_signature(self):
482518

483519
def test_check_signature_invalid_signature(self):
484520
verifier = utils.GitHubTokenScanningPayloadVerifier(
521+
api_url="http://foo",
485522
session=pretend.stub(),
486523
metrics=pretend.stub(),
487524
public_keys_cache=pretend.stub(),
@@ -513,6 +550,7 @@ def test_check_signature_invalid_signature(self):
513550

514551
def test_check_signature_invalid_crypto(self):
515552
verifier = utils.GitHubTokenScanningPayloadVerifier(
553+
api_url="http://foo",
516554
session=pretend.stub(),
517555
metrics=pretend.stub(),
518556
public_keys_cache=pretend.stub(),

‎tests/unit/integration/github/test_views.py

Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,10 @@ def test_github_disclose_token(self, pyramid_request, monkeypatch):
2929

3030
pyramid_request.body = "[1, 2, 3]"
3131
pyramid_request.json_body = [1, 2, 3]
32-
pyramid_request.registry.settings = {"github.token": "token"}
32+
pyramid_request.registry.settings = {
33+
"github.token": "token",
34+
"github.token_scanning_meta_api.url": "http://foo",
35+
}
3336
pyramid_request.find_service = lambda *a, **k: metrics
3437

3538
http = pyramid_request.http = pretend.stub()
@@ -46,7 +49,9 @@ def test_github_disclose_token(self, pyramid_request, monkeypatch):
4649

4750
assert response.status_code == 204
4851
assert verifier_cls.calls == [
49-
pretend.call(session=http, metrics=metrics, api_token="token")
52+
pretend.call(
53+
session=http, metrics=metrics, api_token="token", api_url="http://foo"
54+
)
5055
]
5156
assert verify.calls == [
5257
pretend.call(payload="[1, 2, 3]", key_id="foo", signature="bar")
@@ -70,7 +75,9 @@ def test_github_disclose_token_no_token(self, pyramid_request, monkeypatch):
7075

7176
pyramid_request.body = "[1, 2, 3]"
7277
pyramid_request.json_body = [1, 2, 3]
73-
pyramid_request.registry.settings = {}
78+
pyramid_request.registry.settings = {
79+
"github.token_scanning_meta_api.url": "http://foo"
80+
}
7481
pyramid_request.find_service = lambda *a, **k: metrics
7582
pyramid_request.http = pretend.stub()
7683

@@ -96,7 +103,10 @@ def test_github_disclose_token_verify_fail(self, monkeypatch, pyramid_request):
96103

97104
pyramid_request.body = "[1, 2, 3]"
98105
pyramid_request.find_service = lambda *a, **k: metrics
99-
pyramid_request.registry.settings = {"github.token": "token"}
106+
pyramid_request.registry.settings = {
107+
"github.token": "token",
108+
"github.token_scanning_meta_api.url": "http://foo",
109+
}
100110

101111
pyramid_request.http = pretend.stub()
102112

@@ -137,7 +147,12 @@ def find_service(self, *a, **k):
137147

138148
response = pretend.stub(status_int=200)
139149
http = pretend.stub()
140-
registry = pretend.stub(settings={"github.token": "token"})
150+
registry = pretend.stub(
151+
settings={
152+
"github.token": "token",
153+
"github.token_scanning_meta_api.url": "http://foo",
154+
}
155+
)
141156

142157
request = Request()
143158
response = views.github_disclose_token(request)
@@ -160,7 +175,10 @@ def metrics_increment(key):
160175

161176
pyramid_request.body = "{}"
162177
pyramid_request.json_body = {}
163-
pyramid_request.registry.settings = {"github.token": "token"}
178+
pyramid_request.registry.settings = {
179+
"github.token": "token",
180+
"github.token_scanning_meta_api.url": "http://foo",
181+
}
164182
pyramid_request.find_service = lambda *a, **k: metrics_service
165183

166184
pyramid_request.http = pretend.stub()

‎tests/unit/test_config.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -231,6 +231,9 @@ def __init__(self):
231231
"token.default.max_age": 21600,
232232
"warehouse.xmlrpc.client.ratelimit_string": "3600 per hour",
233233
"warehouse.xmlrpc.search.enabled": True,
234+
"github.token_scanning_meta_api.url": (
235+
"https://github.com/api/meta/public_keys/token_scanning"
236+
),
234237
}
235238
if environment == config.Environment.development:
236239
expected_settings.update(

‎warehouse/config.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,12 @@ def configure(settings=None):
164164
settings, "warehouse.release_files_table", "WAREHOUSE_RELEASE_FILES_TABLE"
165165
)
166166
maybe_set(settings, "github.token", "GITHUB_TOKEN")
167+
maybe_set(
168+
settings,
169+
"github.token_scanning_meta_api.url",
170+
"GITHUB_TOKEN_SCANNING_META_API_URL",
171+
default="https://github.com/api/meta/public_keys/token_scanning",
172+
)
167173
maybe_set(settings, "warehouse.trending_table", "WAREHOUSE_TRENDING_TABLE")
168174
maybe_set(settings, "celery.broker_url", "BROKER_URL")
169175
maybe_set(settings, "celery.result_url", "REDIS_URL")

‎warehouse/integrations/github/utils.py

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -164,13 +164,15 @@ def __init__(
164164
*,
165165
session,
166166
metrics,
167+
api_url: str,
167168
api_token: Optional[str] = None,
168169
public_keys_cache=PUBLIC_KEYS_CACHE,
169170
):
170171
self._metrics = metrics
171172
self._session = session
172173
self._api_token = api_token
173174
self._public_keys_cache = public_keys_cache
175+
self._api_url = api_url
174176

175177
def verify(self, *, payload, key_id, signature):
176178

@@ -217,15 +219,8 @@ def _headers_auth(self):
217219
return {"Authorization": f"token {self._api_token}"}
218220

219221
def _retrieve_public_key_payload(self):
220-
221-
token_scanning_pubkey_api_url = (
222-
"https://github.com/api/meta/public_keys/token_scanning"
223-
)
224-
225222
try:
226-
response = self._session.get(
227-
token_scanning_pubkey_api_url, headers=self._headers_auth()
228-
)
223+
response = self._session.get(self._api_url, headers=self._headers_auth())
229224
response.raise_for_status()
230225
return response.json()
231226
except requests.HTTPError as exc:

‎warehouse/integrations/github/views.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ def github_disclose_token(request):
4747
verifier = utils.GitHubTokenScanningPayloadVerifier(
4848
session=request.http,
4949
metrics=metrics,
50+
api_url=request.registry.settings["github.token_scanning_meta_api.url"],
5051
api_token=request.registry.settings.get("github.token"),
5152
)
5253

0 commit comments

Comments
 (0)