diff --git a/.gitignore b/.gitignore index 34de0fcf..2c654dc1 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,3 @@ DerivedData/ Package.resolved .swiftpm /out - diff --git a/Benchmarks/.gitignore b/Benchmarks/.gitignore new file mode 100644 index 00000000..2517bcdf --- /dev/null +++ b/Benchmarks/.gitignore @@ -0,0 +1,9 @@ +.DS_Store +/.build +/Packages +xcuserdata/ +DerivedData/ +.swiftpm/configuration/registries.json +.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata +.netrc +.benchmarkBaselines/ \ No newline at end of file diff --git a/Benchmarks/Benchmarks/CertificatesBenchmark/Benchmarks.swift b/Benchmarks/Benchmarks/CertificatesBenchmark/Benchmarks.swift new file mode 100644 index 00000000..3ae5a0ea --- /dev/null +++ b/Benchmarks/Benchmarks/CertificatesBenchmark/Benchmarks.swift @@ -0,0 +1,60 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftCertificates open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftCertificates project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftCertificates project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Benchmark +import Foundation + +let benchmarks = { + Benchmark.defaultConfiguration = .init( + metrics: [ + .mallocCountTotal, + .syscalls, + .readSyscalls, + .writeSyscalls, + .memoryLeaked, + .retainCount, + .releaseCount, + ] + ) + + var configWithoutRetainRelease = Benchmark.defaultConfiguration + configWithoutRetainRelease.metrics.removeAll(where: { $0 == .retainCount || $0 == .releaseCount }) + + // async code is currently still quite flaky in the number of retain/release it does so we don't measure them today + Benchmark("Verifier", configuration: configWithoutRetainRelease) { benchmark in + for _ in benchmark.scaledIterations { + await verifier() + } + } + + Benchmark("Parse WebPKI Roots") { benchmark, run in + for _ in benchmark.scaledIterations { + run() + } + } setup: { + parseWebPKIRootsSetup() + } + + Benchmark("TinyArray non-allocating functions") { benchmark in + for _ in benchmark.scaledIterations { + tinyArrayNonAllocationFunctions() + } + } + + Benchmark("TinyArray.append(_:)") { benchmark in + for _ in benchmark.scaledIterations { + tinyArrayAppend() + } + } +} diff --git a/Benchmarks/Benchmarks/CertificatesBenchmark/ParseWebPKI.swift b/Benchmarks/Benchmarks/CertificatesBenchmark/ParseWebPKI.swift new file mode 100644 index 00000000..cdec0dae --- /dev/null +++ b/Benchmarks/Benchmarks/CertificatesBenchmark/ParseWebPKI.swift @@ -0,0 +1,159 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftCertificates open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftCertificates project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftCertificates project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Benchmark +import X509 +import SwiftASN1 +import Foundation + +public func parseWebPKIRootsSetup() -> () -> Void { + let derEncodedCAs = WebPKI.all.map { try! PEMDocument(pemString: $0).derBytes } + return { + for derEncodedCA in derEncodedCAs { + blackHole(try! Certificate(derEncoded: derEncodedCA).extensions.count) + } + } +} + +enum WebPKI { + static let all = [br, af, cf, dz, de] + static let br = """ + -----BEGIN CERTIFICATE----- + MIIDwzCCAqugAwIBAgIBATANBgkqhkiG9w0BAQsFADCBgjELMAkGA1UEBhMCREUx + KzApBgNVBAoMIlQtU3lzdGVtcyBFbnRlcnByaXNlIFNlcnZpY2VzIEdtYkgxHzAd + BgNVBAsMFlQtU3lzdGVtcyBUcnVzdCBDZW50ZXIxJTAjBgNVBAMMHFQtVGVsZVNl + YyBHbG9iYWxSb290IENsYXNzIDMwHhcNMDgxMDAxMTAyOTU2WhcNMzMxMDAxMjM1 + OTU5WjCBgjELMAkGA1UEBhMCREUxKzApBgNVBAoMIlQtU3lzdGVtcyBFbnRlcnBy + aXNlIFNlcnZpY2VzIEdtYkgxHzAdBgNVBAsMFlQtU3lzdGVtcyBUcnVzdCBDZW50 + ZXIxJTAjBgNVBAMMHFQtVGVsZVNlYyBHbG9iYWxSb290IENsYXNzIDMwggEiMA0G + CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC9dZPwYiJvJK7genasfb3ZJNW4t/zN + 8ELg63iIVl6bmlQdTQyK9tPPcPRStdiTBONGhnFBSivwKixVA9ZIw+A5OO3yXDw/ + RLyTPWGrTs0NvvAgJ1gORH8EGoel15YUNpDQSXuhdfsaa3Ox+M6pCSzyU9XDFES4 + hqX2iys52qMzVNn6chr3IhUciJFrf2blw2qAsCTz34ZFiP0Zf3WHHx+xGwpzJFu5 + ZeAsVMhg02YXP+HMVDNzkQI6pn97djmiH5a2OK61yJN0HZ65tOVgnS9W0eDrXltM + EnAMbEQgqxHY9Bn20pxSN+f6tsIxO0rUFJmtxxr1XV/6B7h8DR/Wgx6zAgMBAAGj + QjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBS1 + A/d2O2GCahKqGFPrAyGUv/7OyjANBgkqhkiG9w0BAQsFAAOCAQEAVj3vlNW92nOy + WL6ukK2YJ5f+AbGwUgC4TeQbIXQbfsDuXmkqJa9c1h3a0nnJ85cp4IaH3gRZD/FZ + 1GSFS5mvJQQeyUapl96Cshtwn5z2r3Ex3XsFpSzTucpH9sry9uetuUg/vBa3wW30 + 6gmv7PO15wWeph6KU1HWk4HMdJP2udqmJQV0eVp+QD6CSyYRMG7hP0HHRwA11fXT + 91Q+gT3aSWqas+8QPebrb9HIIkfLzM8BMZLZGOMivgkeGj5asuRrDFR6fUNOuIml + e9eiPZaGzPImNC1qkp2aGtAw4l1OBLBfiyB+d8E9lYLRRpo7PHi4b6HQDWSieB4p + TpPDpFQUWw== + -----END CERTIFICATE----- + """ + static let af = """ + -----BEGIN CERTIFICATE----- + MIIFtzCCA5+gAwIBAgICBQkwDQYJKoZIhvcNAQEFBQAwRTELMAkGA1UEBhMCQk0x + GTAXBgNVBAoTEFF1b1ZhZGlzIExpbWl0ZWQxGzAZBgNVBAMTElF1b1ZhZGlzIFJv + b3QgQ0EgMjAeFw0wNjExMjQxODI3MDBaFw0zMTExMjQxODIzMzNaMEUxCzAJBgNV + BAYTAkJNMRkwFwYDVQQKExBRdW9WYWRpcyBMaW1pdGVkMRswGQYDVQQDExJRdW9W + YWRpcyBSb290IENBIDIwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCa + GMpLlA0ALa8DKYrwD4HIrkwZhR0In6spRIXzL4GtMh6QRr+jhiYaHv5+HBg6XJxg + Fyo6dIMzMH1hVBHL7avg5tKifvVrbxi3Cgst/ek+7wrGsxDp3MJGF/hd/aTa/55J + WpzmM+Yklvc/ulsrHHo1wtZn/qtmUIttKGAr79dgw8eTvI02kfN/+NsRE8Scd3bB + rrcCaoF6qUWD4gXmuVbBlDePSHFjIuwXZQeVikvfj8ZaCuWw419eaxGrDPmF60Tp + +ARz8un+XJiM9XOva7R+zdRcAitMOeGylZUtQofX1bOQQ7dsE/He3fbE+Ik/0XX1 + ksOR1YqI0JDs3G3eicJlcZaLDQP9nL9bFqyS2+r+eXyt66/3FsvbzSUr5R/7mp/i + Ucw6UwxI5g69ybR2BlLmEROFcmMDBOAENisgGQLodKcftslWZvB1JdxnwQ5hYIiz + PtGo/KPaHbDRsSNU30R2be1B2MGyIrZTHN81Hdyhdyox5C315eXbyOD/5YDXC2Og + /zOhD7osFRXql7PSorW+8oyWHhqPHWykYTe5hnMz15eWniN9gqRMgeKh0bpnX5UH + oycR7hYQe7xFSkyyBNKr79X9DFHOUGoIMfmR2gyPZFwDwzqLID9ujWc9Otb+fVuI + yV77zGHcizN300QyNQliBJIWENieJ0f7OyHj+OsdWwIDAQABo4GwMIGtMA8GA1Ud + EwEB/wQFMAMBAf8wCwYDVR0PBAQDAgEGMB0GA1UdDgQWBBQahGK8SEwzJQTU7tD2 + A8QZRtGUazBuBgNVHSMEZzBlgBQahGK8SEwzJQTU7tD2A8QZRtGUa6FJpEcwRTEL + MAkGA1UEBhMCQk0xGTAXBgNVBAoTEFF1b1ZhZGlzIExpbWl0ZWQxGzAZBgNVBAMT + ElF1b1ZhZGlzIFJvb3QgQ0EgMoICBQkwDQYJKoZIhvcNAQEFBQADggIBAD4KFk2f + BluornFdLwUvZ+YTRYPENvbzwCYMDbVHZF34tHLJRqUDGCdViXh9duqWNIAXINzn + g/iN/Ae42l9NLmeyhP3ZRPx3UIHmfLTJDQtyU/h2BwdBR5YM++CCJpNVjP4iH2Bl + fF/nJrP3MpCYUNQ3cVX2kiF495V5+vgtJodmVjB3pjd4M1IQWK4/YY7yarHvGH5K + WWPKjaJW1acvvFYfzznB4vsKqBUsfU16Y8Zsl0Q80m/DShcK+JDSV6IZUaUtl0Ha + B0+pUNqQjZRG4T7wlP0QADj1O+hA4bRuVhogzG9Yje0uRY/W6ZM/57Es3zrWIozc + hLsib9D45MY56QSIPMO661V6bYCZJPVsAfv4l7CUW+v90m/xd2gNNWQjrLhVoQPR + TUIZ3Ph1WVaj+ahJefivDrkRoHy3au000LYmYjgahwz46P0u05B/B5EqHdZ+XIWD + mbA4CD/pXvk1B+TJYm5Xf6dQlfe6yJvmjqIBxdZmv3lh8zwc4bmCXF2gw+nYSL0Z + ohEUGW6yhhtoPkg3Goi3XZZenMfvJ2II4pEZXNLxId26F0KCl3GBUzGpn/Z9Yr9y + 4aOTHcyKJloJONDO1w2AFrR4pTqHTI2KpdVGl/IsELm8VCLAAVBpQ570su9t+Oza + 8eOx79+Rj1QqCyXBJhnEUhAFZdWCEOrCMc0u + -----END CERTIFICATE----- + """ + static let cf = """ + -----BEGIN CERTIFICATE----- + MIICRjCCAc2gAwIBAgIQC6Fa+h3foLVJRK/NJKBs7DAKBggqhkjOPQQDAzBlMQsw + CQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cu + ZGlnaWNlcnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJvb3Qg + RzMwHhcNMTMwODAxMTIwMDAwWhcNMzgwMTE1MTIwMDAwWjBlMQswCQYDVQQGEwJV + UzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNlcnQu + Y29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJvb3QgRzMwdjAQBgcq + hkjOPQIBBgUrgQQAIgNiAAQZ57ysRGXtzbg/WPuNsVepRC0FFfLvC/8QdJ+1YlJf + Zn4f5dwbRXkLzMZTCp2NXQLZqVneAlr2lSoOjThKiknGvMYDOAdfVdp+CW7if17Q + RSAPWXYQ1qAk8C3eNvJsKTmjQjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/ + BAQDAgGGMB0GA1UdDgQWBBTL0L2p4ZgFUaFNN6KDec6NHSrkhDAKBggqhkjOPQQD + AwNnADBkAjAlpIFFAmsSS3V0T8gj43DydXLefInwz5FyYZ5eEJJZVrmDxxDnOOlY + JjZ91eQ0hjkCMHw2U/Aw5WJjOpnitqM7mzT6HtoQknFekROn3aRukswy1vUhZscv + 6pZjamVFkpUBtA== + -----END CERTIFICATE----- + """ + static let dz = """ + -----BEGIN CERTIFICATE----- + MIICYDCCAgegAwIBAgIMDWpfCD8oXD5Rld9dMAoGCCqGSM49BAMCMIGRMQswCQYD + VQQGEwJVUzERMA8GA1UECBMISWxsaW5vaXMxEDAOBgNVBAcTB0NoaWNhZ28xITAf + BgNVBAoTGFRydXN0d2F2ZSBIb2xkaW5ncywgSW5jLjE6MDgGA1UEAxMxVHJ1c3R3 + YXZlIEdsb2JhbCBFQ0MgUDI1NiBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTAeFw0x + NzA4MjMxOTM1MTBaFw00MjA4MjMxOTM1MTBaMIGRMQswCQYDVQQGEwJVUzERMA8G + A1UECBMISWxsaW5vaXMxEDAOBgNVBAcTB0NoaWNhZ28xITAfBgNVBAoTGFRydXN0 + d2F2ZSBIb2xkaW5ncywgSW5jLjE6MDgGA1UEAxMxVHJ1c3R3YXZlIEdsb2JhbCBF + Q0MgUDI1NiBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTBZMBMGByqGSM49AgEGCCqG + SM49AwEHA0IABH77bOYj43MyCMpg5lOcunSNGLB4kFKA3TjASh3RqMyTpJcGOMoN + FWLGjgEqZZ2q3zSRLoHB5DOSMcT9CTqmP62jQzBBMA8GA1UdEwEB/wQFMAMBAf8w + DwYDVR0PAQH/BAUDAwcGADAdBgNVHQ4EFgQUo0EGrJBt0UrrdaVKEJmzsaGLSvcw + CgYIKoZIzj0EAwIDRwAwRAIgB+ZU2g6gWrKuEZ+Hxbb/ad4lvvigtwjzRM4q3wgh + DDcCIC0mA6AFvWvR9lz4ZcyGbbOcNEhjhAnFjXca4syc4XR7 + -----END CERTIFICATE----- + """ + static let de = """ + -----BEGIN CERTIFICATE----- + MIIFiDCCA3CgAwIBAgIIfQmX/vBH6nowDQYJKoZIhvcNAQELBQAwYjELMAkGA1UE + BhMCQ04xMjAwBgNVBAoMKUdVQU5HIERPTkcgQ0VSVElGSUNBVEUgQVVUSE9SSVRZ + IENPLixMVEQuMR8wHQYDVQQDDBZHRENBIFRydXN0QVVUSCBSNSBST09UMB4XDTE0 + MTEyNjA1MTMxNVoXDTQwMTIzMTE1NTk1OVowYjELMAkGA1UEBhMCQ04xMjAwBgNV + BAoMKUdVQU5HIERPTkcgQ0VSVElGSUNBVEUgQVVUSE9SSVRZIENPLixMVEQuMR8w + HQYDVQQDDBZHRENBIFRydXN0QVVUSCBSNSBST09UMIICIjANBgkqhkiG9w0BAQEF + AAOCAg8AMIICCgKCAgEA2aMW8Mh0dHeb7zMNOwZ+Vfy1YI92hhJCfVZmPoiC7XJj + Dp6L3TQsAlFRwxn9WVSEyfFrs0yw6ehGXTjGoqcuEVe6ghWinI9tsJlKCvLriXBj + TnnEt1u9ol2x8kECK62pOqPseQrsXzrj/e+APK00mxqriCZ7VqKChh/rNYmDf1+u + KU49tm7srsHwJ5uu4/Ts765/94Y9cnrrpftZTqfrlYwiOXnhLQiPzLyRuEH3FMEj + qcOtmkVEs7LXLM3GKeJQEK5cy4KOFxg2fZfmiJqwTTQJ9Cy5WmYqsBebnh52nUpm + MUHfP/vFBu8btn4aRjb3ZGM74zkYI+dndRTVdVeSN72+ahsmUPI2JgaQxXABZG12 + ZuGR224HwGGALrIuL4xwp9E7PLOR5G62xDtw8mySlwnNR30YwPO7ng/Wi64HtloP + zgsMR6flPri9fcebNaBhlzpBdRfMK5Z3KpIhHtmVdiBnaM8Nvd/WHwlqmuLMc3Gk + L30SgLdTMEZeS1SZD2fJpcjyIMGC7J0R38IC+xo70e0gmu9lZJIQDSri3nDxGGeC + jGHeuLzRL5z7D9Ar7Rt2ueQ5Vfj4oR24qoAATILnsn8JuLwwoC8N9VKejveSswoA + HQBUlwbgsQfZxw9cZX08bVlX5O2ljelAU58VS6Bx9hoh49pwBiFYFIeFd3mqgnkC + AwEAAaNCMEAwHQYDVR0OBBYEFOLJQJ9NzuiaoXzPDj9lxSmIahlRMA8GA1UdEwEB + /wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3DQEBCwUAA4ICAQDRSVfg + p8xoWLoBDysZzY2wYUWsEe1jUGn4H3++Fo/9nesLqjJHdtJnJO29fDMylyrHBYZm + DRd9FBUb1Ov9H5r2XpdptxolpAqzkT9fNqyL7FeoPueBihhXOYV0GkLH6VsTX4/5 + COmSdI31R9KrO9b7eGZONn356ZLpBN79SWP8bfsUcZNnL0dKt7n/HipzcEYwv1ry + L3ml4Y0M2fmyYzeMN2WFcGpcWwlyua1jPLHd+PwyvzeG5LuOmCd+uh8W4XAR8gPf + JWIyJyYYMoSf/wA6E7qaTfRPuBRwIrHKK5DOKcFw9C+df/KQHtZa37dG/OaG+svg + IHZ6uqbL9XzeYqWxi+7egmaKTjowHz+Ay60nugxe19CxVsp3cbK1daFQqUBDF8Io + 2c9Si1vIY9RCPqAzekYu9wogRlR+ak8x8YF+QnQ4ZXMn7sZ8uI7XpTrXmKGcjBBV + 09tL7ECQ8s1uV9JiDnxXk7Gnbc2dg7sq5+W2O3FYrf3RRbxake5TFW/TRQl1brqQ + XR4EzzffHqhmsYzmIGrv/EhOdJhCrylvLmrH+33RZjEizIYAfmaDDEL0vTSSwxrq + T8p+ck0LcIymSLumoRT2+1hEmRSuqguTaaApJUqlyyvdimYHFngVV3Eb7PVHhPOe + MTd61X8kreS8/f3MboPoDKi3QWwH3b08hpcv0g== + -----END CERTIFICATE----- + """ +} diff --git a/IntegrationTests/tests_01_allocation_counters/test_01_resources/test_tiny_array_cow_append_contents_of.swift b/Benchmarks/Benchmarks/CertificatesBenchmark/TinyArrayAppend.swift similarity index 63% rename from IntegrationTests/tests_01_allocation_counters/test_01_resources/test_tiny_array_cow_append_contents_of.swift rename to Benchmarks/Benchmarks/CertificatesBenchmark/TinyArrayAppend.swift index 04e49892..114116d0 100644 --- a/IntegrationTests/tests_01_allocation_counters/test_01_resources/test_tiny_array_cow_append_contents_of.swift +++ b/Benchmarks/Benchmarks/CertificatesBenchmark/TinyArrayAppend.swift @@ -12,19 +12,17 @@ // //===----------------------------------------------------------------------===// +import Benchmark import _CertificateInternals -func run(identifier: String) { - measure(identifier: identifier) { - var count = 0 - for _ in 0..<1000 { - var tinyArray = _TinyArray() - for i in 0..<1000 { - tinyArray.append(i) - } - count += tinyArray.count - } - - return count +public func tinyArrayAppend() { + var count = 0 + + var tinyArray = _TinyArray() + for i in 0..<1000 { + tinyArray.append(i) } + count += tinyArray.count + + blackHole(count) } diff --git a/IntegrationTests/tests_01_allocation_counters/test_01_resources/test_tiny_array_non_allocating_operations.swift b/Benchmarks/Benchmarks/CertificatesBenchmark/TinyArrayNonAllocationFunctions.swift similarity index 57% rename from IntegrationTests/tests_01_allocation_counters/test_01_resources/test_tiny_array_non_allocating_operations.swift rename to Benchmarks/Benchmarks/CertificatesBenchmark/TinyArrayNonAllocationFunctions.swift index a0edc802..67f33353 100644 --- a/IntegrationTests/tests_01_allocation_counters/test_01_resources/test_tiny_array_non_allocating_operations.swift +++ b/Benchmarks/Benchmarks/CertificatesBenchmark/TinyArrayNonAllocationFunctions.swift @@ -12,21 +12,16 @@ // //===----------------------------------------------------------------------===// +import Benchmark import _CertificateInternals -func run(identifier: String) { - measure(identifier: identifier) { - var counts = 0 - for _ in 0..<1000 { - counts += _TinyArray(CollectionOfOne(1)).count - - do { - var array = _TinyArray() - array.append(contentsOf: CollectionOfOne(1)) - counts += array.count - } - } - - return counts - } +public func tinyArrayNonAllocationFunctions() { + var counts = 0 + counts += _TinyArray(CollectionOfOne(1)).count + + var array = _TinyArray() + array.append(contentsOf: CollectionOfOne(1)) + counts += array.count + + blackHole(counts) } diff --git a/Benchmarks/Benchmarks/CertificatesBenchmark/VerifierBenchmark.swift b/Benchmarks/Benchmarks/CertificatesBenchmark/VerifierBenchmark.swift new file mode 100644 index 00000000..2dd3f3da --- /dev/null +++ b/Benchmarks/Benchmarks/CertificatesBenchmark/VerifierBenchmark.swift @@ -0,0 +1,666 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftCertificates open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftCertificates project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftCertificates project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Benchmark +import X509 +import Foundation +import Crypto +import SwiftASN1 + +public func verifier() async { + var counts = 0 + + counts += await testAllSuccessfulValidations() + counts += await testAllUnsuccessfulValidations() + + blackHole(counts) +} + +// MARK: - successful validation + +func testAllSuccessfulValidations() async -> Int { + var counts = 0 + counts += await testTrivialChainBuilding() + counts += await testExtraRootsAreIgnored() + counts += await testPuttingRootsInTheIntermediariesIsntAProblem() + counts += await testSupportsCrossSignedRootWithoutTrouble() + counts += await testBuildsTheShorterPathInTheCaseOfCrossSignedRoots() + counts += await testPrefersToUseIntermediatesWithSKIThatMatches() + counts += await testPrefersNoSKIToNonMatchingSKI() + counts += await testRejectsRootsThatDidNotSignTheCertBeforeThem() + counts += await testPolicyFailuresCanFindLongerPaths() + counts += await testSelfSignedCertsAreTrustedWhenInTrustStore() + counts += await testTrustRootsCanBeNonSelfSignedLeaves() + counts += await testTrustRootsCanBeNonSelfSignedIntermediates() + return counts +} + +func testTrivialChainBuilding() async -> Int { + let roots = CertificateStore([TestCertificate.ca1]) + + var verifier = Verifier(rootCertificates: roots) { + RFC5280Policy(validationTime: TestCertificate.referenceTime) + } + let result = await verifier.validate( + leafCertificate: TestCertificate.localhostLeaf, + intermediates: CertificateStore([TestCertificate.intermediate1]) + ) + + guard case .validCertificate(let chain) = result else { + fatalError("Failed to validate: \(result)") + } + + return chain.count +} + +func testExtraRootsAreIgnored() async -> Int { + let roots = CertificateStore([TestCertificate.ca1, TestCertificate.ca2]) + + var verifier = Verifier(rootCertificates: roots) { RFC5280Policy(validationTime: TestCertificate.referenceTime) } + let result = await verifier.validate( + leafCertificate: TestCertificate.localhostLeaf, + intermediates: CertificateStore([TestCertificate.intermediate1]) + ) + + guard case .validCertificate(let chain) = result else { + fatalError("Failed to validate: \(result)") + } + + return chain.count +} + +func testPuttingRootsInTheIntermediariesIsntAProblem() async -> Int { + let roots = CertificateStore([TestCertificate.ca1, TestCertificate.ca2]) + + var verifier = Verifier(rootCertificates: roots) { RFC5280Policy(validationTime: TestCertificate.referenceTime) } + let result = await verifier.validate( + leafCertificate: TestCertificate.localhostLeaf, + intermediates: CertificateStore([TestCertificate.intermediate1, TestCertificate.ca1, TestCertificate.ca2]) + ) + + guard case .validCertificate(let chain) = result else { + fatalError("Failed to validate: \(result)") + } + + return chain.count +} + +func testSupportsCrossSignedRootWithoutTrouble() async -> Int { + let roots = CertificateStore([TestCertificate.ca2]) + + var verifier = Verifier(rootCertificates: roots) { RFC5280Policy(validationTime: TestCertificate.referenceTime) } + let result = await verifier.validate( + leafCertificate: TestCertificate.localhostLeaf, + intermediates: CertificateStore([TestCertificate.intermediate1, TestCertificate.ca1CrossSignedByCA2]) + ) + + guard case .validCertificate(let chain) = result else { + fatalError("Failed to validate: \(result)") + } + + return chain.count +} + +func testBuildsTheShorterPathInTheCaseOfCrossSignedRoots() async -> Int { + let roots = CertificateStore([TestCertificate.ca1, TestCertificate.ca2]) + + var verifier = Verifier(rootCertificates: roots) { RFC5280Policy(validationTime: TestCertificate.referenceTime) } + let result = await verifier.validate( + leafCertificate: TestCertificate.localhostLeaf, + intermediates: CertificateStore([ + TestCertificate.intermediate1, TestCertificate.ca2CrossSignedByCA1, TestCertificate.ca1CrossSignedByCA2, + ]) + ) + + guard case .validCertificate(let chain) = result else { + fatalError("Failed to validate: \(result)") + } + + return chain.count +} + +func testPrefersToUseIntermediatesWithSKIThatMatches() async -> Int { + let roots = CertificateStore([TestCertificate.ca1]) + + var verifier = Verifier(rootCertificates: roots) { RFC5280Policy(validationTime: TestCertificate.referenceTime) } + let result = await verifier.validate( + leafCertificate: TestCertificate.localhostLeaf, + intermediates: CertificateStore([TestCertificate.intermediate1, TestCertificate.intermediate1WithoutSKIAKI]) + ) + + guard case .validCertificate(let chain) = result else { + fatalError("Failed to validate: \(result)") + } + + return chain.count +} + +func testPrefersNoSKIToNonMatchingSKI() async -> Int { + let roots = CertificateStore([TestCertificate.ca1]) + + var verifier = Verifier(rootCertificates: roots) { RFC5280Policy(validationTime: TestCertificate.referenceTime) } + let result = await verifier.validate( + leafCertificate: TestCertificate.localhostLeaf, + intermediates: CertificateStore([ + TestCertificate.intermediate1WithIncorrectSKIAKI, TestCertificate.intermediate1WithoutSKIAKI, + ]) + ) + + guard case .validCertificate(let chain) = result else { + fatalError("Failed to validate: \(result)") + } + + return chain.count +} + +func testRejectsRootsThatDidNotSignTheCertBeforeThem() async -> Int { + let roots = CertificateStore([TestCertificate.ca1WithAlternativePrivateKey, TestCertificate.ca2]) + + var verifier = Verifier(rootCertificates: roots) { RFC5280Policy(validationTime: TestCertificate.referenceTime) } + let result = await verifier.validate( + leafCertificate: TestCertificate.localhostLeaf, + intermediates: CertificateStore([ + TestCertificate.ca1CrossSignedByCA2, TestCertificate.ca2CrossSignedByCA1, TestCertificate.intermediate1, + ]) + ) + + guard case .validCertificate(let chain) = result else { + fatalError("Failed to validate: \(result)") + } + return chain.count +} + +func testPolicyFailuresCanFindLongerPaths() async -> Int { + let roots = CertificateStore([TestCertificate.ca1, TestCertificate.ca2]) + + var verifier = Verifier(rootCertificates: roots) { + FailIfCertInChainPolicy(forbiddenCert: TestCertificate.ca1) + RFC5280Policy(validationTime: TestCertificate.referenceTime) + } + let result = await verifier.validate( + leafCertificate: TestCertificate.localhostLeaf, + intermediates: CertificateStore([ + TestCertificate.intermediate1, TestCertificate.ca2CrossSignedByCA1, TestCertificate.ca1CrossSignedByCA2, + ]) + ) + + guard case .validCertificate(let chain) = result else { + fatalError("Failed to validate: \(result)") + } + + return chain.count +} + +func testSelfSignedCertsAreTrustedWhenInTrustStore() async -> Int { + let roots = CertificateStore([TestCertificate.ca1, TestCertificate.isolatedSelfSignedCert]) + + var verifier = Verifier(rootCertificates: roots) { RFC5280Policy(validationTime: TestCertificate.referenceTime) } + let result = await verifier.validate( + leafCertificate: TestCertificate.isolatedSelfSignedCert, + intermediates: CertificateStore([TestCertificate.intermediate1]) + ) + + guard case .validCertificate(let chain) = result else { + fatalError("Failed to validate: \(result)") + } + + return chain.count +} + +func testTrustRootsCanBeNonSelfSignedLeaves() async -> Int { + // we use a custom policy here to ignore the fact that the basic constraints extension is critical. + struct IgnoreBasicConstraintsPolicy: VerifierPolicy { + let verifyingCriticalExtensions: [ASN1ObjectIdentifier] = [.X509ExtensionID.basicConstraints] + + func chainMeetsPolicyRequirements(chain: UnverifiedCertificateChain) async -> PolicyEvaluationResult { + return .meetsPolicy + } + } + + let roots = CertificateStore([TestCertificate.localhostLeaf]) + + var verifier = Verifier(rootCertificates: roots) { IgnoreBasicConstraintsPolicy() } + let result = await verifier.validate( + leafCertificate: TestCertificate.localhostLeaf, + intermediates: CertificateStore([TestCertificate.intermediate1]) + ) + + guard case .validCertificate(let chain) = result else { + fatalError("Failed to validate: \(result)") + } + + return chain.count +} + +func testTrustRootsCanBeNonSelfSignedIntermediates() async -> Int { + let roots = CertificateStore([TestCertificate.intermediate1]) + + var verifier = Verifier(rootCertificates: roots) { RFC5280Policy(validationTime: TestCertificate.referenceTime) } + let result = await verifier.validate( + leafCertificate: TestCertificate.localhostLeaf, + intermediates: CertificateStore([TestCertificate.intermediate1]) + ) + + guard case .validCertificate(let chain) = result else { + fatalError("Failed to validate: \(result)") + } + + return chain.count +} + +// MARK: - unsuccessful validation + +func testAllUnsuccessfulValidations() async -> Int { + var counts = 0 + counts += await testWePoliceCriticalExtensionsOnLeafCerts() + counts += await testMissingIntermediateFailsToBuild() + counts += await testSelfSignedCertsAreRejectedWhenNotInTheTrustStore() + counts += await testMissingRootFailsToBuild() + return counts +} + +func testWePoliceCriticalExtensionsOnLeafCerts() async -> Int { + let roots = CertificateStore([ + TestCertificate.ca1, TestCertificate.isolatedSelfSignedCertWithWeirdCriticalExtension, + ]) + + var verifier = Verifier(rootCertificates: roots) { RFC5280Policy(validationTime: TestCertificate.referenceTime) } + let result = await verifier.validate( + leafCertificate: TestCertificate.isolatedSelfSignedCertWithWeirdCriticalExtension, + intermediates: CertificateStore([TestCertificate.intermediate1]) + ) + + guard case .couldNotValidate(let policyResults) = result else { + fatalError("Incorrectly validated: \(result)") + } + + return policyResults.count +} + +func testMissingIntermediateFailsToBuild() async -> Int { + let roots = CertificateStore([TestCertificate.ca1]) + + var verifier = Verifier(rootCertificates: roots) { RFC5280Policy(validationTime: TestCertificate.referenceTime) } + let result = await verifier.validate( + leafCertificate: TestCertificate.localhostLeaf, + intermediates: CertificateStore([]) + ) + + guard case .couldNotValidate(let policyResults) = result else { + fatalError("Accidentally validated: \(result)") + } + + return policyResults.count +} + +func testSelfSignedCertsAreRejectedWhenNotInTheTrustStore() async -> Int { + let roots = CertificateStore([TestCertificate.ca1]) + + var verifier = Verifier(rootCertificates: roots) { RFC5280Policy(validationTime: TestCertificate.referenceTime) } + let result = await verifier.validate( + leafCertificate: TestCertificate.isolatedSelfSignedCert, + intermediates: CertificateStore([TestCertificate.intermediate1]) + ) + + guard case .couldNotValidate(let policyResults) = result else { + fatalError("Incorrectly validated: \(result)") + } + return policyResults.count +} + +func testMissingRootFailsToBuild() async -> Int { + let roots = CertificateStore([]) + + var verifier = Verifier(rootCertificates: roots) { RFC5280Policy(validationTime: TestCertificate.referenceTime) } + let result = await verifier.validate( + leafCertificate: TestCertificate.localhostLeaf, + intermediates: CertificateStore([TestCertificate.intermediate1]) + ) + + guard case .couldNotValidate(let policyResults) = result else { + fatalError("Accidentally validated: \(result)") + } + + return policyResults.count +} + +private struct FailIfCertInChainPolicy: VerifierPolicy { + let verifyingCriticalExtensions: [ASN1ObjectIdentifier] = [] + + private let forbiddenCert: Certificate + + init(forbiddenCert: Certificate) { + self.forbiddenCert = forbiddenCert + } + + func chainMeetsPolicyRequirements(chain: UnverifiedCertificateChain) async -> PolicyEvaluationResult { + guard chain.contains(self.forbiddenCert) else { + return .meetsPolicy + } + return .failsToMeetPolicy(reason: "chain must not contain \(self.forbiddenCert)") + } +} + +enum TestCertificate { + static let referenceTime = Date() + + static let all = [ + ca1, + ca1CrossSignedByCA2, + ca1WithAlternativePrivateKey, + ca2, + ca2CrossSignedByCA1, + intermediate1, + intermediate1WithoutSKIAKI, + intermediate1WithIncorrectSKIAKI, + localhostLeaf, + isolatedSelfSignedCert, + isolatedSelfSignedCertWithWeirdCriticalExtension, + ] + + private static let ca1PrivateKey = P384.Signing.PrivateKey() + private static let ca1Name = try! DistinguishedName { + CountryName("US") + OrganizationName("Apple") + CommonName("Swift Certificate Test CA 1") + } + static let ca1: Certificate = { + return try! Certificate( + version: .v3, + serialNumber: .init(), + publicKey: .init(ca1PrivateKey.publicKey), + notValidBefore: referenceTime - .days(365), + notValidAfter: referenceTime + .days(3650), + issuer: ca1Name, + subject: ca1Name, + signatureAlgorithm: .ecdsaWithSHA384, + extensions: Certificate.Extensions { + Critical( + BasicConstraints.isCertificateAuthority(maxPathLength: nil) + ) + KeyUsage(keyCertSign: true) + SubjectKeyIdentifier( + keyIdentifier: ArraySlice(Insecure.SHA1.hash(data: ca1PrivateKey.publicKey.derRepresentation)) + ) + }, + issuerPrivateKey: .init(ca1PrivateKey) + ) + }() + static let ca1CrossSignedByCA2: Certificate = { + return try! Certificate( + version: .v3, + serialNumber: .init(), + publicKey: .init(ca1PrivateKey.publicKey), + notValidBefore: referenceTime - .days(365), + notValidAfter: referenceTime + .days(365), + issuer: ca2Name, + subject: ca1Name, + signatureAlgorithm: .ecdsaWithSHA384, + extensions: Certificate.Extensions { + Critical( + BasicConstraints.isCertificateAuthority(maxPathLength: nil) + ) + KeyUsage(keyCertSign: true) + AuthorityKeyIdentifier(keyIdentifier: try! ca2.extensions.subjectKeyIdentifier!.keyIdentifier) + SubjectKeyIdentifier( + keyIdentifier: ArraySlice(Insecure.SHA1.hash(data: ca1PrivateKey.publicKey.derRepresentation)) + ) + }, + issuerPrivateKey: .init(ca2PrivateKey) + ) + }() + private static let ca1AlternativePrivateKey = P384.Signing.PrivateKey() + static let ca1WithAlternativePrivateKey: Certificate = { + return try! Certificate( + version: .v3, + serialNumber: .init(), + publicKey: .init(ca1AlternativePrivateKey.publicKey), + notValidBefore: referenceTime - .days(365), + notValidAfter: referenceTime + .days(3650), + issuer: ca1Name, + subject: ca1Name, + signatureAlgorithm: .ecdsaWithSHA384, + extensions: Certificate.Extensions { + Critical( + BasicConstraints.isCertificateAuthority(maxPathLength: nil) + ) + KeyUsage(keyCertSign: true) + SubjectKeyIdentifier( + keyIdentifier: ArraySlice( + Insecure.SHA1.hash(data: ca1AlternativePrivateKey.publicKey.derRepresentation) + ) + ) + }, + issuerPrivateKey: .init(ca1PrivateKey) + ) + }() + + private static let ca2PrivateKey = P384.Signing.PrivateKey() + private static let ca2Name = try! DistinguishedName { + CountryName("US") + OrganizationName("Apple") + CommonName("Swift Certificate Test CA 2") + } + static let ca2: Certificate = { + return try! Certificate( + version: .v3, + serialNumber: .init(), + publicKey: .init(ca2PrivateKey.publicKey), + notValidBefore: referenceTime - .days(365), + notValidAfter: referenceTime + .days(3650), + issuer: ca2Name, + subject: ca2Name, + signatureAlgorithm: .ecdsaWithSHA384, + extensions: Certificate.Extensions { + Critical( + BasicConstraints.isCertificateAuthority(maxPathLength: nil) + ) + KeyUsage(keyCertSign: true) + SubjectKeyIdentifier( + keyIdentifier: ArraySlice(Insecure.SHA1.hash(data: ca2PrivateKey.publicKey.derRepresentation)) + ) + }, + issuerPrivateKey: .init(ca2PrivateKey) + ) + }() + static let ca2CrossSignedByCA1: Certificate = { + return try! Certificate( + version: .v3, + serialNumber: .init(), + publicKey: .init(ca2PrivateKey.publicKey), + notValidBefore: referenceTime - .days(365), + notValidAfter: referenceTime + .days(365), + issuer: ca1Name, + subject: ca2Name, + signatureAlgorithm: .ecdsaWithSHA384, + extensions: Certificate.Extensions { + Critical( + BasicConstraints.isCertificateAuthority(maxPathLength: nil) + ) + KeyUsage(keyCertSign: true) + AuthorityKeyIdentifier(keyIdentifier: try! ca1.extensions.subjectKeyIdentifier!.keyIdentifier) + SubjectKeyIdentifier( + keyIdentifier: ArraySlice(Insecure.SHA1.hash(data: ca2PrivateKey.publicKey.derRepresentation)) + ) + }, + issuerPrivateKey: .init(ca1PrivateKey) + ) + }() + + static let intermediate1PrivateKey = P256.Signing.PrivateKey() + static let intermediate1Name = try! DistinguishedName { + CountryName("US") + OrganizationName("Apple") + CommonName("Swift Certificate Test Intermediate CA 1") + } + static let intermediate1: Certificate = { + return try! Certificate( + version: .v3, + serialNumber: .init(), + publicKey: .init(intermediate1PrivateKey.publicKey), + notValidBefore: referenceTime - .days(365), + notValidAfter: referenceTime + .days(5 * 365), + issuer: ca1.subject, + subject: intermediate1Name, + signatureAlgorithm: .ecdsaWithSHA384, + extensions: Certificate.Extensions { + Critical( + BasicConstraints.isCertificateAuthority(maxPathLength: 1) + ) + KeyUsage(keyCertSign: true) + AuthorityKeyIdentifier(keyIdentifier: try! ca1.extensions.subjectKeyIdentifier!.keyIdentifier) + SubjectKeyIdentifier( + keyIdentifier: ArraySlice( + Insecure.SHA1.hash(data: intermediate1PrivateKey.publicKey.derRepresentation) + ) + ) + }, + issuerPrivateKey: .init(ca1PrivateKey) + ) + }() + static let intermediate1WithoutSKIAKI: Certificate = { + return try! Certificate( + version: .v3, + serialNumber: .init(), + publicKey: .init(intermediate1PrivateKey.publicKey), + notValidBefore: referenceTime - .days(365), + notValidAfter: referenceTime + .days(5 * 365), + issuer: ca1.subject, + subject: intermediate1Name, + signatureAlgorithm: .ecdsaWithSHA384, + extensions: Certificate.Extensions { + Critical( + BasicConstraints.isCertificateAuthority(maxPathLength: 1) + ) + KeyUsage(keyCertSign: true) + }, + issuerPrivateKey: .init(ca1PrivateKey) + ) + }() + static let intermediate1WithIncorrectSKIAKI: Certificate = { + return try! Certificate( + version: .v3, + serialNumber: .init(), + publicKey: .init(intermediate1PrivateKey.publicKey), + notValidBefore: referenceTime - .days(365), + notValidAfter: referenceTime + .days(5 * 365), + issuer: ca1.subject, + subject: intermediate1Name, + signatureAlgorithm: .ecdsaWithSHA384, + extensions: Certificate.Extensions { + Critical( + BasicConstraints.isCertificateAuthority(maxPathLength: 1) + ) + KeyUsage(keyCertSign: true) + AuthorityKeyIdentifier(keyIdentifier: try! ca2.extensions.subjectKeyIdentifier!.keyIdentifier) + SubjectKeyIdentifier( + keyIdentifier: ArraySlice(Insecure.SHA1.hash(data: ca1PrivateKey.publicKey.derRepresentation)) + ) + }, + issuerPrivateKey: .init(ca1PrivateKey) + ) + }() + + private static let localhostLeafPrivateKey = P256.Signing.PrivateKey() + static let localhostLeaf: Certificate = { + let localhostLeafName = try! DistinguishedName { + CountryName("US") + OrganizationName("Apple") + CommonName("localhost") + } + + return try! Certificate( + version: .v3, + serialNumber: .init(), + publicKey: .init(localhostLeafPrivateKey.publicKey), + notValidBefore: referenceTime - .days(365), + notValidAfter: referenceTime + .days(365), + issuer: intermediate1.subject, + subject: localhostLeafName, + signatureAlgorithm: .ecdsaWithSHA256, + extensions: Certificate.Extensions { + Critical( + BasicConstraints.notCertificateAuthority + ) + KeyUsage(keyCertSign: true) + AuthorityKeyIdentifier(keyIdentifier: try! intermediate1.extensions.subjectKeyIdentifier!.keyIdentifier) + }, + issuerPrivateKey: .init(intermediate1PrivateKey) + ) + }() + + private static let isolatedSelfSignedCertKey = P256.Signing.PrivateKey() + static let isolatedSelfSignedCert: Certificate = { + let isolatedSelfSignedCertName = try! DistinguishedName { + CountryName("US") + OrganizationName("Apple") + CommonName("Isolated Self-Signed Cert") + } + + return try! Certificate( + version: .v3, + serialNumber: .init(), + publicKey: .init(isolatedSelfSignedCertKey.publicKey), + notValidBefore: referenceTime - .days(365), + notValidAfter: referenceTime + .days(365), + issuer: isolatedSelfSignedCertName, + subject: isolatedSelfSignedCertName, + signatureAlgorithm: .ecdsaWithSHA256, + extensions: Certificate.Extensions { + Critical( + BasicConstraints.isCertificateAuthority(maxPathLength: nil) + ) + KeyUsage(keyCertSign: true) + }, + issuerPrivateKey: .init(isolatedSelfSignedCertKey) + ) + }() + + static let isolatedSelfSignedCertWithWeirdCriticalExtension: Certificate = { + let isolatedSelfSignedCertName = try! DistinguishedName { + CountryName("US") + OrganizationName("Apple") + CommonName("Isolated Self-Signed Cert") + } + + return try! Certificate( + version: .v3, + serialNumber: .init(), + publicKey: .init(isolatedSelfSignedCertKey.publicKey), + notValidBefore: referenceTime - .days(365), + notValidAfter: referenceTime + .days(365), + issuer: isolatedSelfSignedCertName, + subject: isolatedSelfSignedCertName, + signatureAlgorithm: .ecdsaWithSHA256, + extensions: Certificate.Extensions { + Critical( + BasicConstraints.isCertificateAuthority(maxPathLength: nil) + ) + KeyUsage(keyCertSign: true) + + // An opaque extension that just so happens to be critical + Certificate.Extension(oid: [1, 2, 3, 4, 5], critical: true, value: [1, 2, 3, 4, 5]) + }, + issuerPrivateKey: .init(isolatedSelfSignedCertKey) + ) + }() +} + +extension TimeInterval { + private static let oneDay: TimeInterval = 60 * 60 * 24 + + static func days(_ days: Int) -> TimeInterval { + return Double(days) * oneDay + } +} diff --git a/Benchmarks/Package.swift b/Benchmarks/Package.swift new file mode 100644 index 00000000..f2dad563 --- /dev/null +++ b/Benchmarks/Package.swift @@ -0,0 +1,44 @@ +// swift-tools-version: 5.7 +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftCertificates open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftCertificates project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftCertificates project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import PackageDescription + +let package = Package( + name: "benchmarks", + platforms: [ + .macOS(.v13), + ], + dependencies: [ + .package(path: "../"), + .package(url: "https://github.com/ordo-one/package-benchmark", from: "1.11.1"), + .package(url: "https://github.com/apple/swift-crypto.git", from: "2.5.0"), + .package(url: "https://github.com/apple/swift-asn1.git", from: "1.0.0-beta.1"), + ], + targets: [ + .executableTarget( + name: "CertificatesBenchmark", + dependencies: [ + .product(name: "Benchmark", package: "package-benchmark"), + .product(name: "X509", package: "swift-certificates"), + .product(name: "SwiftASN1", package: "swift-asn1"), + .product(name: "Crypto", package: "swift-crypto"), + ], + path: "Benchmarks/CertificatesBenchmark", + plugins: [ + .plugin(name: "BenchmarkPlugin", package: "package-benchmark") + ] + ), + ] +) diff --git a/Benchmarks/Thresholds/5.7/CertificatesBenchmark.Parse_WebPKI_Roots.p90.json b/Benchmarks/Thresholds/5.7/CertificatesBenchmark.Parse_WebPKI_Roots.p90.json new file mode 100644 index 00000000..18c7a1ea --- /dev/null +++ b/Benchmarks/Thresholds/5.7/CertificatesBenchmark.Parse_WebPKI_Roots.p90.json @@ -0,0 +1,8 @@ +{ + "mallocCountTotal" : 197, + "memoryLeaked" : 0, + "readSyscalls" : 0, + "releaseCount" : 7418, + "retainCount" : 6982, + "writeSyscalls" : 0 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/5.7/CertificatesBenchmark.TinyArray.append(_:).p90.json b/Benchmarks/Thresholds/5.7/CertificatesBenchmark.TinyArray.append(_:).p90.json new file mode 100644 index 00000000..dcdfad82 --- /dev/null +++ b/Benchmarks/Thresholds/5.7/CertificatesBenchmark.TinyArray.append(_:).p90.json @@ -0,0 +1,8 @@ +{ + "mallocCountTotal" : 10, + "memoryLeaked" : 0, + "readSyscalls" : 0, + "releaseCount" : 12, + "retainCount" : 0, + "writeSyscalls" : 0 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/5.7/CertificatesBenchmark.TinyArray_non-allocating_functions.p90.json b/Benchmarks/Thresholds/5.7/CertificatesBenchmark.TinyArray_non-allocating_functions.p90.json new file mode 100644 index 00000000..bd773412 --- /dev/null +++ b/Benchmarks/Thresholds/5.7/CertificatesBenchmark.TinyArray_non-allocating_functions.p90.json @@ -0,0 +1,8 @@ +{ + "mallocCountTotal" : 0, + "memoryLeaked" : 0, + "readSyscalls" : 0, + "releaseCount" : 1, + "retainCount" : 0, + "writeSyscalls" : 0 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/5.7/CertificatesBenchmark.Verifier.p90.json b/Benchmarks/Thresholds/5.7/CertificatesBenchmark.Verifier.p90.json new file mode 100644 index 00000000..61a21692 --- /dev/null +++ b/Benchmarks/Thresholds/5.7/CertificatesBenchmark.Verifier.p90.json @@ -0,0 +1,6 @@ +{ + "mallocCountTotal" : 1234, + "memoryLeaked" : 0, + "readSyscalls" : 0, + "writeSyscalls" : 0 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/5.8/CertificatesBenchmark.Parse_WebPKI_Roots.p90.json b/Benchmarks/Thresholds/5.8/CertificatesBenchmark.Parse_WebPKI_Roots.p90.json new file mode 100644 index 00000000..67baa2f2 --- /dev/null +++ b/Benchmarks/Thresholds/5.8/CertificatesBenchmark.Parse_WebPKI_Roots.p90.json @@ -0,0 +1,8 @@ +{ + "mallocCountTotal" : 197, + "memoryLeaked" : 0, + "readSyscalls" : 0, + "releaseCount" : 6960, + "retainCount" : 6574, + "writeSyscalls" : 0 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/5.8/CertificatesBenchmark.TinyArray.append(_:).p90.json b/Benchmarks/Thresholds/5.8/CertificatesBenchmark.TinyArray.append(_:).p90.json new file mode 100644 index 00000000..dcdfad82 --- /dev/null +++ b/Benchmarks/Thresholds/5.8/CertificatesBenchmark.TinyArray.append(_:).p90.json @@ -0,0 +1,8 @@ +{ + "mallocCountTotal" : 10, + "memoryLeaked" : 0, + "readSyscalls" : 0, + "releaseCount" : 12, + "retainCount" : 0, + "writeSyscalls" : 0 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/5.8/CertificatesBenchmark.TinyArray_non-allocating_functions.p90.json b/Benchmarks/Thresholds/5.8/CertificatesBenchmark.TinyArray_non-allocating_functions.p90.json new file mode 100644 index 00000000..bd773412 --- /dev/null +++ b/Benchmarks/Thresholds/5.8/CertificatesBenchmark.TinyArray_non-allocating_functions.p90.json @@ -0,0 +1,8 @@ +{ + "mallocCountTotal" : 0, + "memoryLeaked" : 0, + "readSyscalls" : 0, + "releaseCount" : 1, + "retainCount" : 0, + "writeSyscalls" : 0 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/5.8/CertificatesBenchmark.Verifier.p90.json b/Benchmarks/Thresholds/5.8/CertificatesBenchmark.Verifier.p90.json new file mode 100644 index 00000000..61a21692 --- /dev/null +++ b/Benchmarks/Thresholds/5.8/CertificatesBenchmark.Verifier.p90.json @@ -0,0 +1,6 @@ +{ + "mallocCountTotal" : 1234, + "memoryLeaked" : 0, + "readSyscalls" : 0, + "writeSyscalls" : 0 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/5.9/CertificatesBenchmark.Parse_WebPKI_Roots.p90.json b/Benchmarks/Thresholds/5.9/CertificatesBenchmark.Parse_WebPKI_Roots.p90.json new file mode 100644 index 00000000..52255701 --- /dev/null +++ b/Benchmarks/Thresholds/5.9/CertificatesBenchmark.Parse_WebPKI_Roots.p90.json @@ -0,0 +1,8 @@ +{ + "mallocCountTotal" : 197, + "memoryLeaked" : 0, + "readSyscalls" : 0, + "releaseCount" : 6760, + "retainCount" : 6374, + "writeSyscalls" : 0 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/5.9/CertificatesBenchmark.TinyArray.append(_:).p90.json b/Benchmarks/Thresholds/5.9/CertificatesBenchmark.TinyArray.append(_:).p90.json new file mode 100644 index 00000000..dcdfad82 --- /dev/null +++ b/Benchmarks/Thresholds/5.9/CertificatesBenchmark.TinyArray.append(_:).p90.json @@ -0,0 +1,8 @@ +{ + "mallocCountTotal" : 10, + "memoryLeaked" : 0, + "readSyscalls" : 0, + "releaseCount" : 12, + "retainCount" : 0, + "writeSyscalls" : 0 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/5.9/CertificatesBenchmark.TinyArray_non-allocating_functions.p90.json b/Benchmarks/Thresholds/5.9/CertificatesBenchmark.TinyArray_non-allocating_functions.p90.json new file mode 100644 index 00000000..bd773412 --- /dev/null +++ b/Benchmarks/Thresholds/5.9/CertificatesBenchmark.TinyArray_non-allocating_functions.p90.json @@ -0,0 +1,8 @@ +{ + "mallocCountTotal" : 0, + "memoryLeaked" : 0, + "readSyscalls" : 0, + "releaseCount" : 1, + "retainCount" : 0, + "writeSyscalls" : 0 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/5.9/CertificatesBenchmark.Verifier.p90.json b/Benchmarks/Thresholds/5.9/CertificatesBenchmark.Verifier.p90.json new file mode 100644 index 00000000..42f48a8d --- /dev/null +++ b/Benchmarks/Thresholds/5.9/CertificatesBenchmark.Verifier.p90.json @@ -0,0 +1,6 @@ +{ + "mallocCountTotal" : 1233, + "memoryLeaked" : 0, + "readSyscalls" : 0, + "writeSyscalls" : 0 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/main/CertificatesBenchmark.Parse_WebPKI_Roots.p90.json b/Benchmarks/Thresholds/main/CertificatesBenchmark.Parse_WebPKI_Roots.p90.json new file mode 100644 index 00000000..caca0609 --- /dev/null +++ b/Benchmarks/Thresholds/main/CertificatesBenchmark.Parse_WebPKI_Roots.p90.json @@ -0,0 +1,8 @@ +{ + "mallocCountTotal" : 197, + "memoryLeaked" : 0, + "readSyscalls" : 0, + "releaseCount" : 4199, + "retainCount" : 3775, + "writeSyscalls" : 0 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/main/CertificatesBenchmark.TinyArray.append(_:).p90.json b/Benchmarks/Thresholds/main/CertificatesBenchmark.TinyArray.append(_:).p90.json new file mode 100644 index 00000000..dcdfad82 --- /dev/null +++ b/Benchmarks/Thresholds/main/CertificatesBenchmark.TinyArray.append(_:).p90.json @@ -0,0 +1,8 @@ +{ + "mallocCountTotal" : 10, + "memoryLeaked" : 0, + "readSyscalls" : 0, + "releaseCount" : 12, + "retainCount" : 0, + "writeSyscalls" : 0 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/main/CertificatesBenchmark.TinyArray_non-allocating_functions.p90.json b/Benchmarks/Thresholds/main/CertificatesBenchmark.TinyArray_non-allocating_functions.p90.json new file mode 100644 index 00000000..8e9ac913 --- /dev/null +++ b/Benchmarks/Thresholds/main/CertificatesBenchmark.TinyArray_non-allocating_functions.p90.json @@ -0,0 +1,8 @@ +{ + "mallocCountTotal" : 0, + "memoryLeaked" : 0, + "readSyscalls" : 0, + "releaseCount" : 0, + "retainCount" : 0, + "writeSyscalls" : 0 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/main/CertificatesBenchmark.Verifier.p90.json b/Benchmarks/Thresholds/main/CertificatesBenchmark.Verifier.p90.json new file mode 100644 index 00000000..42f48a8d --- /dev/null +++ b/Benchmarks/Thresholds/main/CertificatesBenchmark.Verifier.p90.json @@ -0,0 +1,6 @@ +{ + "mallocCountTotal" : 1233, + "memoryLeaked" : 0, + "readSyscalls" : 0, + "writeSyscalls" : 0 +} \ No newline at end of file diff --git a/IntegrationTests/plugin_echo.sh b/IntegrationTests/plugin_echo.sh deleted file mode 100644 index 77fdbb7b..00000000 --- a/IntegrationTests/plugin_echo.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -##===----------------------------------------------------------------------===## -## -## This source file is part of the SwiftCertificates open source project -## -## Copyright (c) 2023 Apple Inc. and the SwiftCertificates project authors -## Licensed under Apache License v2.0 -## -## See LICENSE.txt for license information -## See CONTRIBUTORS.txt for the list of SwiftCertificates project authors -## -## SPDX-License-Identifier: Apache-2.0 -## -##===----------------------------------------------------------------------===## - -function plugin_echo_test_suite_begin() { - echo "Running test suite '$1'" -} - -function plugin_echo_test_suite_end() { - true -} - -# test_name -function plugin_echo_test_begin() { - echo -n "Running test '$1'... " -} - -function plugin_echo_test_skip() { - echo "Skipping test '$1'" -} - -function plugin_echo_test_ok() { - echo "OK (${1}s)" -} - -function plugin_echo_test_fail() { - echo "FAILURE ($1)" - echo "--- OUTPUT BEGIN ---" - cat "$2" - echo "--- OUTPUT END ---" -} - -function plugin_echo_test_end() { - true -} - -function plugin_echo_summary_ok() { - echo "OK (ran $1 tests successfully)" -} - -function plugin_echo_summary_fail() { - echo "FAILURE (oks: $1, failures: $2)" -} - -function plugin_echo_init() { - true -} diff --git a/IntegrationTests/plugin_junit_xml.sh b/IntegrationTests/plugin_junit_xml.sh deleted file mode 100644 index 7c9070d0..00000000 --- a/IntegrationTests/plugin_junit_xml.sh +++ /dev/null @@ -1,119 +0,0 @@ -#!/bin/bash -##===----------------------------------------------------------------------===## -## -## This source file is part of the SwiftCertificates open source project -## -## Copyright (c) 2023 Apple Inc. and the SwiftCertificates project authors -## Licensed under Apache License v2.0 -## -## See LICENSE.txt for license information -## See CONTRIBUTORS.txt for the list of SwiftCertificates project authors -## -## SPDX-License-Identifier: Apache-2.0 -## -##===----------------------------------------------------------------------===## - -junit_testsuite_time=0 - -function junit_output_write() { - extra_flags="" - if [[ "$1" == "-n" ]]; then - extra_flags="-n" - shift - fi - test -n "$junit_xml_output" - echo $extra_flags "$*" >> "$junit_xml_output" -} - -function junit_output_cat() { - cat "$@" >> "$junit_xml_output" -} - -# search, replace -function junit_output_replace() { - test -n "$junit_xml_output" - case "$(uname -s)" in - Linux) - sed -i "s/$1/$2/g" "$junit_xml_output" - ;; - *) - sed -i "" "s/$1/$2/g" "$junit_xml_output" - ;; - esac -} - -function plugin_junit_xml_test_suite_begin() { - junit_testsuite_time=0 - junit_output_write "" -} - -function plugin_junit_xml_test_suite_end() { - junit_repl_success_and_fail "$1" "$2" - junit_output_write "" -} - -# test_name -function plugin_junit_xml_test_begin() { - junit_output_write -n " " - junit_testsuite_time=$((junit_testsuite_time + time_ms)) -} - -function plugin_junit_xml_test_fail() { - time_ms=$1 - junit_output_write " time='$time_ms'>" - junit_output_write " " - junit_output_write " " - junit_output_write ' ' - junit_output_write " " - junit_output_write " " -} - -function plugin_junit_xml_test_end() { - junit_output_write " " -} - -function junit_repl_success_and_fail() { - junit_output_replace XXX-TESTS-XXX "$(($1 + $2))" - junit_output_replace XXX-FAILURES-XXX "$2" - junit_output_replace XXX-TIME-XXX "$junit_testsuite_time" -} - -function plugin_junit_xml_summary_ok() { - junit_output_write "" -} - -function plugin_junit_xml_summary_fail() { - junit_output_write "" -} - -function plugin_junit_xml_init() { - junit_xml_output="" - for f in "$@"; do - if [[ "$junit_xml_output" = "PLACEHOLDER" ]]; then - junit_xml_output="$f" - fi - if [[ "$f" == "--junit-xml" && -z "$junit_xml_output" ]]; then - junit_xml_output="PLACEHOLDER" - fi - done - - if [[ -z "$junit_xml_output" || "$junit_xml_output" = "PLACEHOLDER" ]]; then - echo >&2 "ERROR: you need to specify the output after the --junit-xml argument" - false - fi - echo "" > "$junit_xml_output" -} diff --git a/IntegrationTests/run-tests.sh b/IntegrationTests/run-tests.sh deleted file mode 100755 index a3decafe..00000000 --- a/IntegrationTests/run-tests.sh +++ /dev/null @@ -1,159 +0,0 @@ -#!/bin/bash -##===----------------------------------------------------------------------===## -## -## This source file is part of the SwiftCertificates open source project -## -## Copyright (c) 2023 Apple Inc. and the SwiftCertificates project authors -## Licensed under Apache License v2.0 -## -## See LICENSE.txt for license information -## See CONTRIBUTORS.txt for the list of SwiftCertificates project authors -## -## SPDX-License-Identifier: Apache-2.0 -## -##===----------------------------------------------------------------------===## - -set -eu - -shopt -s nullglob - -here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -tmp=$(mktemp -d /tmp/.swift-certificates-sh-tests_XXXXXX) - -# start_time -function time_diff_to_now() { - echo "$(( $(date +%s) - $1 ))" -} - -function plugins_do() { - local method - method="$1" - shift - for plugin in $plugins; do - cd "$orig_cwd" - "plugin_${plugin}_${method}" "$@" - cd - > /dev/null - done -} - -source "$here/plugin_echo.sh" -source "$here/plugin_junit_xml.sh" - -plugins="echo" -plugin_opts_ind=0 -if [[ "${1-default}" == "--junit-xml" ]]; then - plugins="echo junit_xml" - plugin_opts_ind=2 -fi - -function usage() { - echo >&2 "Usage: $0 [OPTIONS]" - echo >&2 - echo >&2 "OPTIONS:" - echo >&2 " -f FILTER: Only run tests matching FILTER (regex)" -} - -orig_cwd=$(pwd) -cd "$here" - -plugins_do init "$@" -shift $plugin_opts_ind - -filter="." -verbose=false -show_info=false -debug=false -while getopts "f:vid" opt; do - case $opt in - f) - filter="$OPTARG" - ;; - v) - verbose=true - ;; - i) - show_info=true - ;; - d) - debug=true - ;; - \?) - usage - exit 1 - ;; - esac -done - -function run_test() { - if $verbose; then - "$@" 2>&1 | tee -a "$out" - # we need to return the return value of the first command - return ${PIPESTATUS[0]} - else - "$@" >> "$out" 2>&1 - fi -} - -exec 3>&1 4>&2 # copy stdout/err to fd 3/4 to we can output control messages -cnt_ok=0 -cnt_fail=0 -for f in tests_*; do - suite_ok=0 - suite_fail=0 - plugins_do test_suite_begin "$f" - start_suite=$(date +%s) - cd "$f" - for t in test_*.sh; do - if [[ ! "$f/$t" =~ $filter ]]; then - plugins_do test_skip "$t" - continue - fi - out=$(mktemp "$tmp/test.out_XXXXXX") - test_tmp=$(mktemp -d "$tmp/test.tmp_XXXXXX") - plugins_do test_begin "$t" "$f" - start=$(date +%s) - if run_test "$here/run-single-test.sh" "$here/$f/$t" "$test_tmp" "$here/.." "$show_info"; then - plugins_do test_ok "$(time_diff_to_now $start)" - suite_ok=$((suite_ok+1)) - if $verbose; then - cat "$out" - fi - else - plugins_do test_fail "$(time_diff_to_now $start)" "$out" - suite_fail=$((suite_fail+1)) - fi - if ! $debug; then - rm "$out" - rm -rf "$test_tmp" - fi - plugins_do test_end - done - cnt_ok=$((cnt_ok + suite_ok)) - cnt_fail=$((cnt_fail + suite_fail)) - cd .. - plugins_do test_suite_end "$(time_diff_to_now $start_suite)" "$suite_ok" "$suite_fail" -done - -if ! $debug; then - rm -rf "$tmp" -else - echo >&2 "debug mode, not deleting '$tmp'" -fi - - -# report -if [[ $cnt_fail > 0 ]]; then - # kill leftovers (the whole process group) - trap '' TERM - kill 0 - - plugins_do summary_fail "$cnt_ok" "$cnt_fail" -else - plugins_do summary_ok "$cnt_ok" "$cnt_fail" -fi - -if [[ $cnt_fail > 0 ]]; then - exit 1 -else - exit 0 -fi diff --git a/IntegrationTests/test_functions.sh b/IntegrationTests/test_functions.sh deleted file mode 100644 index 6fb05282..00000000 --- a/IntegrationTests/test_functions.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash -##===----------------------------------------------------------------------===## -## -## This source file is part of the SwiftCertificates open source project -## -## Copyright (c) 2023 Apple Inc. and the SwiftCertificates project authors -## Licensed under Apache License v2.0 -## -## See LICENSE.txt for license information -## See CONTRIBUTORS.txt for the list of SwiftCertificates project authors -## -## SPDX-License-Identifier: Apache-2.0 -## -##===----------------------------------------------------------------------===## -function fail() { - echo >&2 "FAILURE: $*" - false -} - -function assert_equal() { - if [[ "$1" != "$2" ]]; then - fail "expected '$1', got '$2' ${3-}" - fi -} - -function assert_equal_files() { - if ! cmp -s "$1" "$2"; then - diff -u "$1" "$2" || true - echo - echo "--- SNIP ($1, size=$(wc "$1"), SHA=$(shasum "$1")) ---" - cat "$1" - echo "--- SNAP ($1)---" - echo "--- SNIP ($2, size=$(wc "$2"), SHA=$(shasum "$2")) ---" - cat "$2" - echo "--- SNAP ($2) ---" - fail "file '$1' not equal to '$2'" - fi -} - -function assert_less_than() { - if [[ ! "$1" -lt "$2" ]]; then - fail "assertion '$1' < '$2' failed" - fi -} - -function assert_less_than_or_equal() { - if [[ ! "$1" -le "$2" ]]; then - fail "assertion '$1' <= '$2' failed" - fi -} - -function assert_greater_than() { - if [[ ! "$1" -gt "$2" ]]; then - fail "assertion '$1' > '$2' failed" - fi -} - -function assert_greater_than_or_equal() { - if [[ ! "$1" -ge "$2" ]]; then - fail "assertion '$1' >= '$2' failed" - fi -} - -g_has_previously_infoed=false - -function info() { - if $g_show_info; then - if ! $g_has_previously_infoed; then - echo >&3 || true # echo an extra newline so it looks better - g_has_previously_infoed=true - fi - echo >&3 "info: $*" || true - fi -} - -function warn() { - echo >&4 "warning: $*" -} diff --git a/IntegrationTests/tests_01_allocation_counters/defines.sh b/IntegrationTests/tests_01_allocation_counters/defines.sh deleted file mode 100644 index 8df520f8..00000000 --- a/IntegrationTests/tests_01_allocation_counters/defines.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -##===----------------------------------------------------------------------===## -## -## This source file is part of the SwiftCertificates open source project -## -## Copyright (c) 2023 Apple Inc. and the SwiftCertificates project authors -## Licensed under Apache License v2.0 -## -## See LICENSE.txt for license information -## See CONTRIBUTORS.txt for the list of SwiftCertificates project authors -## -## SPDX-License-Identifier: Apache-2.0 -## -##===----------------------------------------------------------------------===## diff --git a/IntegrationTests/tests_01_allocation_counters/test_01_allocation_counts.sh b/IntegrationTests/tests_01_allocation_counters/test_01_allocation_counts.sh deleted file mode 100644 index c752feb6..00000000 --- a/IntegrationTests/tests_01_allocation_counters/test_01_allocation_counts.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash -##===----------------------------------------------------------------------===## -## -## This source file is part of the SwiftCertificates open source project -## -## Copyright (c) 2023 Apple Inc. and the SwiftCertificates project authors -## Licensed under Apache License v2.0 -## -## See LICENSE.txt for license information -## See CONTRIBUTORS.txt for the list of SwiftCertificates project authors -## -## SPDX-License-Identifier: Apache-2.0 -## -##===----------------------------------------------------------------------===## - -source defines.sh - -set -eu -here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -all_tests=() -for file in "$here/test_01_resources/"test_*.swift; do - test_name=$(basename "$file") - test_name=${test_name#test_*} - test_name=${test_name%*.swift} - all_tests+=( "$test_name" ) -done - -"$here/test_01_resources/run-swift-certificates-alloc-counter-tests.sh" -t "$tmp" > "$tmp/output" - -for test in "${all_tests[@]}"; do - cat "$tmp/output" # helps debugging - - while read -r test_case; do - test_case=${test_case#test_*} - total_allocations=$(grep "^test_$test_case.total_allocations:" "$tmp/output" | cut -d: -f2 | sed 's/ //g') - not_freed_allocations=$(grep "^test_$test_case.remaining_allocations:" "$tmp/output" | cut -d: -f2 | sed 's/ //g') - max_allowed_env_name="MAX_ALLOCS_ALLOWED_$test_case" - - info "$test_case: allocations not freed: $not_freed_allocations" - info "$test_case: total number of mallocs: $total_allocations" - - assert_less_than "$not_freed_allocations" 5 # allow some slack - assert_greater_than "$not_freed_allocations" -5 # allow some slack - if [[ -z "${!max_allowed_env_name+x}" ]]; then - if [[ -z "${!max_allowed_env_name+x}" ]]; then - warn "no reference number of allocations set (set to \$$max_allowed_env_name)" - warn "to set current number:" - warn " export $max_allowed_env_name=$total_allocations" - fi - else - max_allowed=${!max_allowed_env_name} - assert_less_than_or_equal "$total_allocations" "$max_allowed" - assert_greater_than "$total_allocations" "$(( max_allowed - 1000))" - fi - done < <(grep "^test_$test[^\W]*.total_allocations:" "$tmp/output" | cut -d: -f1 | cut -d. -f1 | sort | uniq) -done diff --git a/IntegrationTests/tests_01_allocation_counters/test_01_resources/run-swift-certificates-alloc-counter-tests.sh b/IntegrationTests/tests_01_allocation_counters/test_01_resources/run-swift-certificates-alloc-counter-tests.sh deleted file mode 100755 index 39561ca0..00000000 --- a/IntegrationTests/tests_01_allocation_counters/test_01_resources/run-swift-certificates-alloc-counter-tests.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash -##===----------------------------------------------------------------------===## -## -## This source file is part of the SwiftCertificates open source project -## -## Copyright (c) 2023 Apple Inc. and the SwiftCertificates project authors -## Licensed under Apache License v2.0 -## -## See LICENSE.txt for license information -## See CONTRIBUTORS.txt for the list of SwiftCertificates project authors -## -## SPDX-License-Identifier: Apache-2.0 -## -##===----------------------------------------------------------------------===## - -set -eu -here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -tmp_dir="/tmp" - -while getopts "t:" opt; do - case "$opt" in - t) - tmp_dir="$OPTARG" - ;; - *) - exit 1 - ;; - esac -done - -nio_checkout=$(mktemp -d "$tmp_dir/.swift-nio_XXXXXX") -( -cd "$nio_checkout" -git clone --depth 1 https://github.com/apple/swift-nio -) - -shift $((OPTIND-1)) - -tests_to_run=("$here"/test_*.swift) - -if [[ $# -gt 0 ]]; then - tests_to_run=("$@") -fi - -"$nio_checkout/swift-nio/IntegrationTests/allocation-counter-tests-framework/run-allocation-counter.sh" \ - -p "$here/../../.." \ - -m SwiftASN1 \ - -m X509 \ - -m Crypto \ - -s "$here/shared.swift" \ - -t "$tmp_dir" \ - -d <( echo '.package(url: "https://github.com/apple/swift-asn1.git", from: "1.0.0-beta.1"),.package(url: "https://github.com/apple/swift-crypto.git", from: "2.5.0"),' ) \ - "${tests_to_run[@]}" diff --git a/IntegrationTests/tests_01_allocation_counters/test_01_resources/shared.swift b/IntegrationTests/tests_01_allocation_counters/test_01_resources/shared.swift deleted file mode 100644 index 943fb236..00000000 --- a/IntegrationTests/tests_01_allocation_counters/test_01_resources/shared.swift +++ /dev/null @@ -1,447 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the SwiftCertificates open source project -// -// Copyright (c) 2023 Apple Inc. and the SwiftCertificates project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of SwiftCertificates project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// - - -import X509 -import Foundation -import Crypto - -enum WebPKI { - static let all = [br, af, cf, dz, de] - static let br = """ - -----BEGIN CERTIFICATE----- - MIIDwzCCAqugAwIBAgIBATANBgkqhkiG9w0BAQsFADCBgjELMAkGA1UEBhMCREUx - KzApBgNVBAoMIlQtU3lzdGVtcyBFbnRlcnByaXNlIFNlcnZpY2VzIEdtYkgxHzAd - BgNVBAsMFlQtU3lzdGVtcyBUcnVzdCBDZW50ZXIxJTAjBgNVBAMMHFQtVGVsZVNl - YyBHbG9iYWxSb290IENsYXNzIDMwHhcNMDgxMDAxMTAyOTU2WhcNMzMxMDAxMjM1 - OTU5WjCBgjELMAkGA1UEBhMCREUxKzApBgNVBAoMIlQtU3lzdGVtcyBFbnRlcnBy - aXNlIFNlcnZpY2VzIEdtYkgxHzAdBgNVBAsMFlQtU3lzdGVtcyBUcnVzdCBDZW50 - ZXIxJTAjBgNVBAMMHFQtVGVsZVNlYyBHbG9iYWxSb290IENsYXNzIDMwggEiMA0G - CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC9dZPwYiJvJK7genasfb3ZJNW4t/zN - 8ELg63iIVl6bmlQdTQyK9tPPcPRStdiTBONGhnFBSivwKixVA9ZIw+A5OO3yXDw/ - RLyTPWGrTs0NvvAgJ1gORH8EGoel15YUNpDQSXuhdfsaa3Ox+M6pCSzyU9XDFES4 - hqX2iys52qMzVNn6chr3IhUciJFrf2blw2qAsCTz34ZFiP0Zf3WHHx+xGwpzJFu5 - ZeAsVMhg02YXP+HMVDNzkQI6pn97djmiH5a2OK61yJN0HZ65tOVgnS9W0eDrXltM - EnAMbEQgqxHY9Bn20pxSN+f6tsIxO0rUFJmtxxr1XV/6B7h8DR/Wgx6zAgMBAAGj - QjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBS1 - A/d2O2GCahKqGFPrAyGUv/7OyjANBgkqhkiG9w0BAQsFAAOCAQEAVj3vlNW92nOy - WL6ukK2YJ5f+AbGwUgC4TeQbIXQbfsDuXmkqJa9c1h3a0nnJ85cp4IaH3gRZD/FZ - 1GSFS5mvJQQeyUapl96Cshtwn5z2r3Ex3XsFpSzTucpH9sry9uetuUg/vBa3wW30 - 6gmv7PO15wWeph6KU1HWk4HMdJP2udqmJQV0eVp+QD6CSyYRMG7hP0HHRwA11fXT - 91Q+gT3aSWqas+8QPebrb9HIIkfLzM8BMZLZGOMivgkeGj5asuRrDFR6fUNOuIml - e9eiPZaGzPImNC1qkp2aGtAw4l1OBLBfiyB+d8E9lYLRRpo7PHi4b6HQDWSieB4p - TpPDpFQUWw== - -----END CERTIFICATE----- - """ - static let af = """ - -----BEGIN CERTIFICATE----- - MIIFtzCCA5+gAwIBAgICBQkwDQYJKoZIhvcNAQEFBQAwRTELMAkGA1UEBhMCQk0x - GTAXBgNVBAoTEFF1b1ZhZGlzIExpbWl0ZWQxGzAZBgNVBAMTElF1b1ZhZGlzIFJv - b3QgQ0EgMjAeFw0wNjExMjQxODI3MDBaFw0zMTExMjQxODIzMzNaMEUxCzAJBgNV - BAYTAkJNMRkwFwYDVQQKExBRdW9WYWRpcyBMaW1pdGVkMRswGQYDVQQDExJRdW9W - YWRpcyBSb290IENBIDIwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCa - GMpLlA0ALa8DKYrwD4HIrkwZhR0In6spRIXzL4GtMh6QRr+jhiYaHv5+HBg6XJxg - Fyo6dIMzMH1hVBHL7avg5tKifvVrbxi3Cgst/ek+7wrGsxDp3MJGF/hd/aTa/55J - WpzmM+Yklvc/ulsrHHo1wtZn/qtmUIttKGAr79dgw8eTvI02kfN/+NsRE8Scd3bB - rrcCaoF6qUWD4gXmuVbBlDePSHFjIuwXZQeVikvfj8ZaCuWw419eaxGrDPmF60Tp - +ARz8un+XJiM9XOva7R+zdRcAitMOeGylZUtQofX1bOQQ7dsE/He3fbE+Ik/0XX1 - ksOR1YqI0JDs3G3eicJlcZaLDQP9nL9bFqyS2+r+eXyt66/3FsvbzSUr5R/7mp/i - Ucw6UwxI5g69ybR2BlLmEROFcmMDBOAENisgGQLodKcftslWZvB1JdxnwQ5hYIiz - PtGo/KPaHbDRsSNU30R2be1B2MGyIrZTHN81Hdyhdyox5C315eXbyOD/5YDXC2Og - /zOhD7osFRXql7PSorW+8oyWHhqPHWykYTe5hnMz15eWniN9gqRMgeKh0bpnX5UH - oycR7hYQe7xFSkyyBNKr79X9DFHOUGoIMfmR2gyPZFwDwzqLID9ujWc9Otb+fVuI - yV77zGHcizN300QyNQliBJIWENieJ0f7OyHj+OsdWwIDAQABo4GwMIGtMA8GA1Ud - EwEB/wQFMAMBAf8wCwYDVR0PBAQDAgEGMB0GA1UdDgQWBBQahGK8SEwzJQTU7tD2 - A8QZRtGUazBuBgNVHSMEZzBlgBQahGK8SEwzJQTU7tD2A8QZRtGUa6FJpEcwRTEL - MAkGA1UEBhMCQk0xGTAXBgNVBAoTEFF1b1ZhZGlzIExpbWl0ZWQxGzAZBgNVBAMT - ElF1b1ZhZGlzIFJvb3QgQ0EgMoICBQkwDQYJKoZIhvcNAQEFBQADggIBAD4KFk2f - BluornFdLwUvZ+YTRYPENvbzwCYMDbVHZF34tHLJRqUDGCdViXh9duqWNIAXINzn - g/iN/Ae42l9NLmeyhP3ZRPx3UIHmfLTJDQtyU/h2BwdBR5YM++CCJpNVjP4iH2Bl - fF/nJrP3MpCYUNQ3cVX2kiF495V5+vgtJodmVjB3pjd4M1IQWK4/YY7yarHvGH5K - WWPKjaJW1acvvFYfzznB4vsKqBUsfU16Y8Zsl0Q80m/DShcK+JDSV6IZUaUtl0Ha - B0+pUNqQjZRG4T7wlP0QADj1O+hA4bRuVhogzG9Yje0uRY/W6ZM/57Es3zrWIozc - hLsib9D45MY56QSIPMO661V6bYCZJPVsAfv4l7CUW+v90m/xd2gNNWQjrLhVoQPR - TUIZ3Ph1WVaj+ahJefivDrkRoHy3au000LYmYjgahwz46P0u05B/B5EqHdZ+XIWD - mbA4CD/pXvk1B+TJYm5Xf6dQlfe6yJvmjqIBxdZmv3lh8zwc4bmCXF2gw+nYSL0Z - ohEUGW6yhhtoPkg3Goi3XZZenMfvJ2II4pEZXNLxId26F0KCl3GBUzGpn/Z9Yr9y - 4aOTHcyKJloJONDO1w2AFrR4pTqHTI2KpdVGl/IsELm8VCLAAVBpQ570su9t+Oza - 8eOx79+Rj1QqCyXBJhnEUhAFZdWCEOrCMc0u - -----END CERTIFICATE----- - """ - static let cf = """ - -----BEGIN CERTIFICATE----- - MIICRjCCAc2gAwIBAgIQC6Fa+h3foLVJRK/NJKBs7DAKBggqhkjOPQQDAzBlMQsw - CQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cu - ZGlnaWNlcnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJvb3Qg - RzMwHhcNMTMwODAxMTIwMDAwWhcNMzgwMTE1MTIwMDAwWjBlMQswCQYDVQQGEwJV - UzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNlcnQu - Y29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJvb3QgRzMwdjAQBgcq - hkjOPQIBBgUrgQQAIgNiAAQZ57ysRGXtzbg/WPuNsVepRC0FFfLvC/8QdJ+1YlJf - Zn4f5dwbRXkLzMZTCp2NXQLZqVneAlr2lSoOjThKiknGvMYDOAdfVdp+CW7if17Q - RSAPWXYQ1qAk8C3eNvJsKTmjQjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/ - BAQDAgGGMB0GA1UdDgQWBBTL0L2p4ZgFUaFNN6KDec6NHSrkhDAKBggqhkjOPQQD - AwNnADBkAjAlpIFFAmsSS3V0T8gj43DydXLefInwz5FyYZ5eEJJZVrmDxxDnOOlY - JjZ91eQ0hjkCMHw2U/Aw5WJjOpnitqM7mzT6HtoQknFekROn3aRukswy1vUhZscv - 6pZjamVFkpUBtA== - -----END CERTIFICATE----- - """ - static let dz = """ - -----BEGIN CERTIFICATE----- - MIICYDCCAgegAwIBAgIMDWpfCD8oXD5Rld9dMAoGCCqGSM49BAMCMIGRMQswCQYD - VQQGEwJVUzERMA8GA1UECBMISWxsaW5vaXMxEDAOBgNVBAcTB0NoaWNhZ28xITAf - BgNVBAoTGFRydXN0d2F2ZSBIb2xkaW5ncywgSW5jLjE6MDgGA1UEAxMxVHJ1c3R3 - YXZlIEdsb2JhbCBFQ0MgUDI1NiBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTAeFw0x - NzA4MjMxOTM1MTBaFw00MjA4MjMxOTM1MTBaMIGRMQswCQYDVQQGEwJVUzERMA8G - A1UECBMISWxsaW5vaXMxEDAOBgNVBAcTB0NoaWNhZ28xITAfBgNVBAoTGFRydXN0 - d2F2ZSBIb2xkaW5ncywgSW5jLjE6MDgGA1UEAxMxVHJ1c3R3YXZlIEdsb2JhbCBF - Q0MgUDI1NiBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTBZMBMGByqGSM49AgEGCCqG - SM49AwEHA0IABH77bOYj43MyCMpg5lOcunSNGLB4kFKA3TjASh3RqMyTpJcGOMoN - FWLGjgEqZZ2q3zSRLoHB5DOSMcT9CTqmP62jQzBBMA8GA1UdEwEB/wQFMAMBAf8w - DwYDVR0PAQH/BAUDAwcGADAdBgNVHQ4EFgQUo0EGrJBt0UrrdaVKEJmzsaGLSvcw - CgYIKoZIzj0EAwIDRwAwRAIgB+ZU2g6gWrKuEZ+Hxbb/ad4lvvigtwjzRM4q3wgh - DDcCIC0mA6AFvWvR9lz4ZcyGbbOcNEhjhAnFjXca4syc4XR7 - -----END CERTIFICATE----- - """ - static let de = """ - -----BEGIN CERTIFICATE----- - MIIFiDCCA3CgAwIBAgIIfQmX/vBH6nowDQYJKoZIhvcNAQELBQAwYjELMAkGA1UE - BhMCQ04xMjAwBgNVBAoMKUdVQU5HIERPTkcgQ0VSVElGSUNBVEUgQVVUSE9SSVRZ - IENPLixMVEQuMR8wHQYDVQQDDBZHRENBIFRydXN0QVVUSCBSNSBST09UMB4XDTE0 - MTEyNjA1MTMxNVoXDTQwMTIzMTE1NTk1OVowYjELMAkGA1UEBhMCQ04xMjAwBgNV - BAoMKUdVQU5HIERPTkcgQ0VSVElGSUNBVEUgQVVUSE9SSVRZIENPLixMVEQuMR8w - HQYDVQQDDBZHRENBIFRydXN0QVVUSCBSNSBST09UMIICIjANBgkqhkiG9w0BAQEF - AAOCAg8AMIICCgKCAgEA2aMW8Mh0dHeb7zMNOwZ+Vfy1YI92hhJCfVZmPoiC7XJj - Dp6L3TQsAlFRwxn9WVSEyfFrs0yw6ehGXTjGoqcuEVe6ghWinI9tsJlKCvLriXBj - TnnEt1u9ol2x8kECK62pOqPseQrsXzrj/e+APK00mxqriCZ7VqKChh/rNYmDf1+u - KU49tm7srsHwJ5uu4/Ts765/94Y9cnrrpftZTqfrlYwiOXnhLQiPzLyRuEH3FMEj - qcOtmkVEs7LXLM3GKeJQEK5cy4KOFxg2fZfmiJqwTTQJ9Cy5WmYqsBebnh52nUpm - MUHfP/vFBu8btn4aRjb3ZGM74zkYI+dndRTVdVeSN72+ahsmUPI2JgaQxXABZG12 - ZuGR224HwGGALrIuL4xwp9E7PLOR5G62xDtw8mySlwnNR30YwPO7ng/Wi64HtloP - zgsMR6flPri9fcebNaBhlzpBdRfMK5Z3KpIhHtmVdiBnaM8Nvd/WHwlqmuLMc3Gk - L30SgLdTMEZeS1SZD2fJpcjyIMGC7J0R38IC+xo70e0gmu9lZJIQDSri3nDxGGeC - jGHeuLzRL5z7D9Ar7Rt2ueQ5Vfj4oR24qoAATILnsn8JuLwwoC8N9VKejveSswoA - HQBUlwbgsQfZxw9cZX08bVlX5O2ljelAU58VS6Bx9hoh49pwBiFYFIeFd3mqgnkC - AwEAAaNCMEAwHQYDVR0OBBYEFOLJQJ9NzuiaoXzPDj9lxSmIahlRMA8GA1UdEwEB - /wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3DQEBCwUAA4ICAQDRSVfg - p8xoWLoBDysZzY2wYUWsEe1jUGn4H3++Fo/9nesLqjJHdtJnJO29fDMylyrHBYZm - DRd9FBUb1Ov9H5r2XpdptxolpAqzkT9fNqyL7FeoPueBihhXOYV0GkLH6VsTX4/5 - COmSdI31R9KrO9b7eGZONn356ZLpBN79SWP8bfsUcZNnL0dKt7n/HipzcEYwv1ry - L3ml4Y0M2fmyYzeMN2WFcGpcWwlyua1jPLHd+PwyvzeG5LuOmCd+uh8W4XAR8gPf - JWIyJyYYMoSf/wA6E7qaTfRPuBRwIrHKK5DOKcFw9C+df/KQHtZa37dG/OaG+svg - IHZ6uqbL9XzeYqWxi+7egmaKTjowHz+Ay60nugxe19CxVsp3cbK1daFQqUBDF8Io - 2c9Si1vIY9RCPqAzekYu9wogRlR+ak8x8YF+QnQ4ZXMn7sZ8uI7XpTrXmKGcjBBV - 09tL7ECQ8s1uV9JiDnxXk7Gnbc2dg7sq5+W2O3FYrf3RRbxake5TFW/TRQl1brqQ - XR4EzzffHqhmsYzmIGrv/EhOdJhCrylvLmrH+33RZjEizIYAfmaDDEL0vTSSwxrq - T8p+ck0LcIymSLumoRT2+1hEmRSuqguTaaApJUqlyyvdimYHFngVV3Eb7PVHhPOe - MTd61X8kreS8/f3MboPoDKi3QWwH3b08hpcv0g== - -----END CERTIFICATE----- - """ -} - - -@available(macOS 11.0, *) -enum TestCertificate { - static let referenceTime = Date() - - static let all = [ - ca1, - ca1CrossSignedByCA2, - ca1WithAlternativePrivateKey, - ca2, - ca2CrossSignedByCA1, - intermediate1, - intermediate1WithoutSKIAKI, - intermediate1WithIncorrectSKIAKI, - localhostLeaf, - isolatedSelfSignedCert, - isolatedSelfSignedCertWithWeirdCriticalExtension, - ] - - private static let ca1PrivateKey = P384.Signing.PrivateKey() - private static let ca1Name = try! DistinguishedName { - CountryName("US") - OrganizationName("Apple") - CommonName("Swift Certificate Test CA 1") - } - static let ca1: Certificate = { - return try! Certificate( - version: .v3, - serialNumber: .init(), - publicKey: .init(ca1PrivateKey.publicKey), - notValidBefore: referenceTime - .days(365), - notValidAfter: referenceTime + .days(3650), - issuer: ca1Name, - subject: ca1Name, - signatureAlgorithm: .ecdsaWithSHA384, - extensions: Certificate.Extensions { - Critical( - BasicConstraints.isCertificateAuthority(maxPathLength: nil) - ) - KeyUsage(keyCertSign: true) - SubjectKeyIdentifier(keyIdentifier: ArraySlice(Insecure.SHA1.hash(data: ca1PrivateKey.publicKey.derRepresentation))) - }, - issuerPrivateKey: .init(ca1PrivateKey) - ) - }() - static let ca1CrossSignedByCA2: Certificate = { - return try! Certificate( - version: .v3, - serialNumber: .init(), - publicKey: .init(ca1PrivateKey.publicKey), - notValidBefore: referenceTime - .days(365), - notValidAfter: referenceTime + .days(365), - issuer: ca2Name, - subject: ca1Name, - signatureAlgorithm: .ecdsaWithSHA384, - extensions: Certificate.Extensions { - Critical( - BasicConstraints.isCertificateAuthority(maxPathLength: nil) - ) - KeyUsage(keyCertSign: true) - AuthorityKeyIdentifier(keyIdentifier: try! ca2.extensions.subjectKeyIdentifier!.keyIdentifier) - SubjectKeyIdentifier(keyIdentifier: ArraySlice(Insecure.SHA1.hash(data: ca1PrivateKey.publicKey.derRepresentation))) - }, - issuerPrivateKey: .init(ca2PrivateKey) - ) - }() - private static let ca1AlternativePrivateKey = P384.Signing.PrivateKey() - static let ca1WithAlternativePrivateKey: Certificate = { - return try! Certificate( - version: .v3, - serialNumber: .init(), - publicKey: .init(ca1AlternativePrivateKey.publicKey), - notValidBefore: referenceTime - .days(365), - notValidAfter: referenceTime + .days(3650), - issuer: ca1Name, - subject: ca1Name, - signatureAlgorithm: .ecdsaWithSHA384, - extensions: Certificate.Extensions { - Critical( - BasicConstraints.isCertificateAuthority(maxPathLength: nil) - ) - KeyUsage(keyCertSign: true) - SubjectKeyIdentifier(keyIdentifier: ArraySlice(Insecure.SHA1.hash(data: ca1AlternativePrivateKey.publicKey.derRepresentation))) - }, - issuerPrivateKey: .init(ca1PrivateKey) - ) - }() - - private static let ca2PrivateKey = P384.Signing.PrivateKey() - private static let ca2Name = try! DistinguishedName { - CountryName("US") - OrganizationName("Apple") - CommonName("Swift Certificate Test CA 2") - } - static let ca2: Certificate = { - return try! Certificate( - version: .v3, - serialNumber: .init(), - publicKey: .init(ca2PrivateKey.publicKey), - notValidBefore: referenceTime - .days(365), - notValidAfter: referenceTime + .days(3650), - issuer: ca2Name, - subject: ca2Name, - signatureAlgorithm: .ecdsaWithSHA384, - extensions: Certificate.Extensions { - Critical( - BasicConstraints.isCertificateAuthority(maxPathLength: nil) - ) - KeyUsage(keyCertSign: true) - SubjectKeyIdentifier(keyIdentifier: ArraySlice(Insecure.SHA1.hash(data: ca2PrivateKey.publicKey.derRepresentation))) - }, - issuerPrivateKey: .init(ca2PrivateKey) - ) - }() - static let ca2CrossSignedByCA1: Certificate = { - return try! Certificate( - version: .v3, - serialNumber: .init(), - publicKey: .init(ca2PrivateKey.publicKey), - notValidBefore: referenceTime - .days(365), - notValidAfter: referenceTime + .days(365), - issuer: ca1Name, - subject: ca2Name, - signatureAlgorithm: .ecdsaWithSHA384, - extensions: Certificate.Extensions { - Critical( - BasicConstraints.isCertificateAuthority(maxPathLength: nil) - ) - KeyUsage(keyCertSign: true) - AuthorityKeyIdentifier(keyIdentifier: try! ca1.extensions.subjectKeyIdentifier!.keyIdentifier) - SubjectKeyIdentifier(keyIdentifier: ArraySlice(Insecure.SHA1.hash(data: ca2PrivateKey.publicKey.derRepresentation))) - }, - issuerPrivateKey: .init(ca1PrivateKey) - ) - }() - - static let intermediate1PrivateKey = P256.Signing.PrivateKey() - static let intermediate1Name = try! DistinguishedName { - CountryName("US") - OrganizationName("Apple") - CommonName("Swift Certificate Test Intermediate CA 1") - } - static let intermediate1: Certificate = { - return try! Certificate( - version: .v3, - serialNumber: .init(), - publicKey: .init(intermediate1PrivateKey.publicKey), - notValidBefore: referenceTime - .days(365), - notValidAfter: referenceTime + .days(5 * 365), - issuer: ca1.subject, - subject: intermediate1Name, - signatureAlgorithm: .ecdsaWithSHA384, - extensions: Certificate.Extensions { - Critical( - BasicConstraints.isCertificateAuthority(maxPathLength: 1) - ) - KeyUsage(keyCertSign: true) - AuthorityKeyIdentifier(keyIdentifier: try! ca1.extensions.subjectKeyIdentifier!.keyIdentifier) - SubjectKeyIdentifier(keyIdentifier: ArraySlice(Insecure.SHA1.hash(data: intermediate1PrivateKey.publicKey.derRepresentation))) - }, - issuerPrivateKey: .init(ca1PrivateKey) - ) - }() - static let intermediate1WithoutSKIAKI: Certificate = { - return try! Certificate( - version: .v3, - serialNumber: .init(), - publicKey: .init(intermediate1PrivateKey.publicKey), - notValidBefore: referenceTime - .days(365), - notValidAfter: referenceTime + .days(5 * 365), - issuer: ca1.subject, - subject: intermediate1Name, - signatureAlgorithm: .ecdsaWithSHA384, - extensions: Certificate.Extensions { - Critical( - BasicConstraints.isCertificateAuthority(maxPathLength: 1) - ) - KeyUsage(keyCertSign: true) - }, - issuerPrivateKey: .init(ca1PrivateKey) - ) - }() - static let intermediate1WithIncorrectSKIAKI: Certificate = { - return try! Certificate( - version: .v3, - serialNumber: .init(), - publicKey: .init(intermediate1PrivateKey.publicKey), - notValidBefore: referenceTime - .days(365), - notValidAfter: referenceTime + .days(5 * 365), - issuer: ca1.subject, - subject: intermediate1Name, - signatureAlgorithm: .ecdsaWithSHA384, - extensions: Certificate.Extensions { - Critical( - BasicConstraints.isCertificateAuthority(maxPathLength: 1) - ) - KeyUsage(keyCertSign: true) - AuthorityKeyIdentifier(keyIdentifier: try! ca2.extensions.subjectKeyIdentifier!.keyIdentifier) - SubjectKeyIdentifier(keyIdentifier: ArraySlice(Insecure.SHA1.hash(data: ca1PrivateKey.publicKey.derRepresentation))) - }, - issuerPrivateKey: .init(ca1PrivateKey) - ) - }() - - private static let localhostLeafPrivateKey = P256.Signing.PrivateKey() - static let localhostLeaf: Certificate = { - let localhostLeafName = try! DistinguishedName { - CountryName("US") - OrganizationName("Apple") - CommonName("localhost") - } - - return try! Certificate( - version: .v3, - serialNumber: .init(), - publicKey: .init(localhostLeafPrivateKey.publicKey), - notValidBefore: referenceTime - .days(365), - notValidAfter: referenceTime + .days(365), - issuer: intermediate1.subject, - subject: localhostLeafName, - signatureAlgorithm: .ecdsaWithSHA256, - extensions: Certificate.Extensions { - Critical( - BasicConstraints.notCertificateAuthority - ) - KeyUsage(keyCertSign: true) - AuthorityKeyIdentifier(keyIdentifier: try! intermediate1.extensions.subjectKeyIdentifier!.keyIdentifier) - }, - issuerPrivateKey: .init(intermediate1PrivateKey) - ) - }() - - private static let isolatedSelfSignedCertKey = P256.Signing.PrivateKey() - static let isolatedSelfSignedCert: Certificate = { - let isolatedSelfSignedCertName = try! DistinguishedName { - CountryName("US") - OrganizationName("Apple") - CommonName("Isolated Self-Signed Cert") - } - - return try! Certificate( - version: .v3, - serialNumber: .init(), - publicKey: .init(isolatedSelfSignedCertKey.publicKey), - notValidBefore: referenceTime - .days(365), - notValidAfter: referenceTime + .days(365), - issuer: isolatedSelfSignedCertName, - subject: isolatedSelfSignedCertName, - signatureAlgorithm: .ecdsaWithSHA256, - extensions: Certificate.Extensions { - Critical( - BasicConstraints.isCertificateAuthority(maxPathLength: nil) - ) - KeyUsage(keyCertSign: true) - }, - issuerPrivateKey: .init(isolatedSelfSignedCertKey) - ) - }() - - static let isolatedSelfSignedCertWithWeirdCriticalExtension: Certificate = { - let isolatedSelfSignedCertName = try! DistinguishedName { - CountryName("US") - OrganizationName("Apple") - CommonName("Isolated Self-Signed Cert") - } - - return try! Certificate( - version: .v3, - serialNumber: .init(), - publicKey: .init(isolatedSelfSignedCertKey.publicKey), - notValidBefore: referenceTime - .days(365), - notValidAfter: referenceTime + .days(365), - issuer: isolatedSelfSignedCertName, - subject: isolatedSelfSignedCertName, - signatureAlgorithm: .ecdsaWithSHA256, - extensions: Certificate.Extensions { - Critical( - BasicConstraints.isCertificateAuthority(maxPathLength: nil) - ) - KeyUsage(keyCertSign: true) - - // An opaque extension that just so happens to be critical - Certificate.Extension(oid: [1, 2, 3, 4, 5], critical: true, value: [1, 2, 3, 4, 5]) - }, - issuerPrivateKey: .init(isolatedSelfSignedCertKey) - ) - }() -} - -extension TimeInterval { - private static let oneDay: TimeInterval = 60 * 60 * 24 - - static func days(_ days: Int) -> TimeInterval { - return Double(days) * oneDay - } -} diff --git a/IntegrationTests/tests_01_allocation_counters/test_01_resources/test_parse_webpki_roots.swift b/IntegrationTests/tests_01_allocation_counters/test_01_resources/test_parse_webpki_roots.swift deleted file mode 100644 index 544207ed..00000000 --- a/IntegrationTests/tests_01_allocation_counters/test_01_resources/test_parse_webpki_roots.swift +++ /dev/null @@ -1,30 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the SwiftCertificates open source project -// -// Copyright (c) 2023 Apple Inc. and the SwiftCertificates project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of SwiftCertificates project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// - -import X509 -import Foundation -import SwiftASN1 - -func run(identifier: String) { - let derEncodedCAs = WebPKI.all.map { try! PEMDocument(pemString: $0).derBytes } - measure(identifier: identifier) { - var totalExtensionCount = 0 - for _ in 0..<1000 { - for derEncodedCA in derEncodedCAs { - totalExtensionCount += try! Certificate(derEncoded: derEncodedCA).extensions.count - } - } - return totalExtensionCount - } -} diff --git a/IntegrationTests/tests_01_allocation_counters/test_01_resources/test_validation.swift b/IntegrationTests/tests_01_allocation_counters/test_01_resources/test_validation.swift deleted file mode 100644 index ca9f0d00..00000000 --- a/IntegrationTests/tests_01_allocation_counters/test_01_resources/test_validation.swift +++ /dev/null @@ -1,321 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the SwiftCertificates open source project -// -// Copyright (c) 2023 Apple Inc. and the SwiftCertificates project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of SwiftCertificates project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// - -import X509 -import Foundation -import Crypto -import SwiftASN1 - - -func run(identifier: String) { - guard #available(macOS 11.0, *) else { return } - var counts = 0 - // this allocates all certificates before we start counting allocations - counts += TestCertificate.all.map { $0.extensions.count }.reduce(0, +) - measure(identifier: identifier) { - - for _ in 0..<100 { - counts += await testAllSuccessfulValidations() - counts += await testAllUnsuccessfulValidations() - } - - return counts - } -} - -// MARK: - successful validation - -@available(macOS 11.0, *) -func testAllSuccessfulValidations() async -> Int { - var counts = 0 - counts += await testTrivialChainBuilding() - counts += await testExtraRootsAreIgnored() - counts += await testPuttingRootsInTheIntermediariesIsntAProblem() - counts += await testSupportsCrossSignedRootWithoutTrouble() - counts += await testBuildsTheShorterPathInTheCaseOfCrossSignedRoots() - counts += await testPrefersToUseIntermediatesWithSKIThatMatches() - counts += await testPrefersNoSKIToNonMatchingSKI() - counts += await testRejectsRootsThatDidNotSignTheCertBeforeThem() - counts += await testPolicyFailuresCanFindLongerPaths() - counts += await testSelfSignedCertsAreTrustedWhenInTrustStore() - counts += await testTrustRootsCanBeNonSelfSignedLeaves() - counts += await testTrustRootsCanBeNonSelfSignedIntermediates() - return counts -} - -@available(macOS 11.0, *) -func testTrivialChainBuilding() async -> Int { - let roots = CertificateStore([TestCertificate.ca1]) - - var verifier = Verifier(rootCertificates: roots) { - RFC5280Policy(validationTime: TestCertificate.referenceTime) - } - let result = await verifier.validate(leafCertificate: TestCertificate.localhostLeaf, intermediates: CertificateStore([TestCertificate.intermediate1])) - - guard case .validCertificate(let chain) = result else { - fatalError("Failed to validate: \(result)") - } - - return chain.count -} - -@available(macOS 11.0, *) -func testExtraRootsAreIgnored() async -> Int { - let roots = CertificateStore([TestCertificate.ca1, TestCertificate.ca2]) - - var verifier = Verifier(rootCertificates: roots) { RFC5280Policy(validationTime: TestCertificate.referenceTime) } - let result = await verifier.validate(leafCertificate: TestCertificate.localhostLeaf, intermediates: CertificateStore([TestCertificate.intermediate1])) - - guard case .validCertificate(let chain) = result else { - fatalError("Failed to validate: \(result)") - } - - return chain.count -} - -@available(macOS 11.0, *) -func testPuttingRootsInTheIntermediariesIsntAProblem() async -> Int { - let roots = CertificateStore([TestCertificate.ca1, TestCertificate.ca2]) - - var verifier = Verifier(rootCertificates: roots) { RFC5280Policy(validationTime: TestCertificate.referenceTime) } - let result = await verifier.validate(leafCertificate: TestCertificate.localhostLeaf, intermediates: CertificateStore([TestCertificate.intermediate1, TestCertificate.ca1, TestCertificate.ca2])) - - guard case .validCertificate(let chain) = result else { - fatalError("Failed to validate: \(result)") - } - - return chain.count -} - -@available(macOS 11.0, *) -func testSupportsCrossSignedRootWithoutTrouble() async -> Int { - let roots = CertificateStore([TestCertificate.ca2]) - - var verifier = Verifier(rootCertificates: roots) { RFC5280Policy(validationTime: TestCertificate.referenceTime) } - let result = await verifier.validate(leafCertificate: TestCertificate.localhostLeaf, intermediates: CertificateStore([TestCertificate.intermediate1, TestCertificate.ca1CrossSignedByCA2])) - - guard case .validCertificate(let chain) = result else { - fatalError("Failed to validate: \(result)") - } - - return chain.count -} - -@available(macOS 11.0, *) -func testBuildsTheShorterPathInTheCaseOfCrossSignedRoots() async -> Int { - let roots = CertificateStore([TestCertificate.ca1, TestCertificate.ca2]) - - var verifier = Verifier(rootCertificates: roots) { RFC5280Policy(validationTime: TestCertificate.referenceTime) } - let result = await verifier.validate(leafCertificate: TestCertificate.localhostLeaf, intermediates: CertificateStore([TestCertificate.intermediate1, TestCertificate.ca2CrossSignedByCA1, TestCertificate.ca1CrossSignedByCA2])) - - guard case .validCertificate(let chain) = result else { - fatalError("Failed to validate: \(result)") - } - - return chain.count -} - -@available(macOS 11.0, *) -func testPrefersToUseIntermediatesWithSKIThatMatches() async -> Int { - let roots = CertificateStore([TestCertificate.ca1]) - - var verifier = Verifier(rootCertificates: roots) { RFC5280Policy(validationTime: TestCertificate.referenceTime) } - let result = await verifier.validate(leafCertificate: TestCertificate.localhostLeaf, intermediates: CertificateStore([TestCertificate.intermediate1, TestCertificate.intermediate1WithoutSKIAKI])) - - guard case .validCertificate(let chain) = result else { - fatalError("Failed to validate: \(result)") - } - - return chain.count -} - -@available(macOS 11.0, *) -func testPrefersNoSKIToNonMatchingSKI() async -> Int { - let roots = CertificateStore([TestCertificate.ca1]) - - var verifier = Verifier(rootCertificates: roots) { RFC5280Policy(validationTime: TestCertificate.referenceTime) } - let result = await verifier.validate(leafCertificate: TestCertificate.localhostLeaf, intermediates: CertificateStore([TestCertificate.intermediate1WithIncorrectSKIAKI, TestCertificate.intermediate1WithoutSKIAKI])) - - guard case .validCertificate(let chain) = result else { - fatalError("Failed to validate: \(result)") - } - - return chain.count -} - -@available(macOS 11.0, *) -func testRejectsRootsThatDidNotSignTheCertBeforeThem() async -> Int { - let roots = CertificateStore([TestCertificate.ca1WithAlternativePrivateKey, TestCertificate.ca2]) - - var verifier = Verifier(rootCertificates: roots) { RFC5280Policy(validationTime: TestCertificate.referenceTime) } - let result = await verifier.validate(leafCertificate: TestCertificate.localhostLeaf, intermediates: CertificateStore([TestCertificate.ca1CrossSignedByCA2, TestCertificate.ca2CrossSignedByCA1, TestCertificate.intermediate1])) - - guard case .validCertificate(let chain) = result else { - fatalError("Failed to validate: \(result)") - } - return chain.count -} - -@available(macOS 11.0, *) -func testPolicyFailuresCanFindLongerPaths() async -> Int { - let roots = CertificateStore([TestCertificate.ca1, TestCertificate.ca2]) - - var verifier = Verifier(rootCertificates: roots) { - FailIfCertInChainPolicy(forbiddenCert: TestCertificate.ca1) - RFC5280Policy(validationTime: TestCertificate.referenceTime) - } - let result = await verifier.validate(leafCertificate: TestCertificate.localhostLeaf, intermediates: CertificateStore([TestCertificate.intermediate1, TestCertificate.ca2CrossSignedByCA1, TestCertificate.ca1CrossSignedByCA2])) - - guard case .validCertificate(let chain) = result else { - fatalError("Failed to validate: \(result)") - } - - return chain.count -} - -@available(macOS 11.0, *) -func testSelfSignedCertsAreTrustedWhenInTrustStore() async -> Int { - let roots = CertificateStore([TestCertificate.ca1, TestCertificate.isolatedSelfSignedCert]) - - var verifier = Verifier(rootCertificates: roots) { RFC5280Policy(validationTime: TestCertificate.referenceTime) } - let result = await verifier.validate(leafCertificate: TestCertificate.isolatedSelfSignedCert, intermediates: CertificateStore([TestCertificate.intermediate1])) - - guard case .validCertificate(let chain) = result else { - fatalError("Failed to validate: \(result)") - } - - return chain.count -} - -@available(macOS 11.0, *) -func testTrustRootsCanBeNonSelfSignedLeaves() async -> Int { - // we use a custom policy here to ignore the fact that the basic constraints extension is critical. - struct IgnoreBasicConstraintsPolicy: VerifierPolicy { - let verifyingCriticalExtensions: [ASN1ObjectIdentifier] = [.X509ExtensionID.basicConstraints] - -func chainMeetsPolicyRequirements(chain: UnverifiedCertificateChain) async -> PolicyEvaluationResult { - return .meetsPolicy - } - } - - let roots = CertificateStore([TestCertificate.localhostLeaf]) - - var verifier = Verifier(rootCertificates: roots) { IgnoreBasicConstraintsPolicy() } - let result = await verifier.validate(leafCertificate: TestCertificate.localhostLeaf, intermediates: CertificateStore([TestCertificate.intermediate1])) - - guard case .validCertificate(let chain) = result else { - fatalError("Failed to validate: \(result)") - } - - return chain.count -} - -@available(macOS 11.0, *) -func testTrustRootsCanBeNonSelfSignedIntermediates() async -> Int { - let roots = CertificateStore([TestCertificate.intermediate1]) - - var verifier = Verifier(rootCertificates: roots) { RFC5280Policy(validationTime: TestCertificate.referenceTime) } - let result = await verifier.validate(leafCertificate: TestCertificate.localhostLeaf, intermediates: CertificateStore([TestCertificate.intermediate1])) - - guard case .validCertificate(let chain) = result else { - fatalError("Failed to validate: \(result)") - } - - return chain.count -} - -// MARK: - unsuccessful validation - -@available(macOS 11.0, *) -func testAllUnsuccessfulValidations() async -> Int { - var counts = 0 - counts += await testWePoliceCriticalExtensionsOnLeafCerts() - counts += await testMissingIntermediateFailsToBuild() - counts += await testSelfSignedCertsAreRejectedWhenNotInTheTrustStore() - counts += await testMissingRootFailsToBuild() - return counts -} - -@available(macOS 11.0, *) -func testWePoliceCriticalExtensionsOnLeafCerts() async -> Int { - let roots = CertificateStore([TestCertificate.ca1, TestCertificate.isolatedSelfSignedCertWithWeirdCriticalExtension]) - - var verifier = Verifier(rootCertificates: roots) { RFC5280Policy(validationTime: TestCertificate.referenceTime) } - let result = await verifier.validate(leafCertificate: TestCertificate.isolatedSelfSignedCertWithWeirdCriticalExtension, intermediates: CertificateStore([TestCertificate.intermediate1])) - - guard case .couldNotValidate(let policyResults) = result else { - fatalError("Incorrectly validated: \(result)") - } - - return policyResults.count -} - -@available(macOS 11.0, *) -func testMissingIntermediateFailsToBuild() async -> Int { - let roots = CertificateStore([TestCertificate.ca1]) - - var verifier = Verifier(rootCertificates: roots) { RFC5280Policy(validationTime: TestCertificate.referenceTime) } - let result = await verifier.validate(leafCertificate: TestCertificate.localhostLeaf, intermediates: CertificateStore([])) - - guard case .couldNotValidate(let policyResults) = result else { - fatalError("Accidentally validated: \(result)") - } - - return policyResults.count -} - -@available(macOS 11.0, *) -func testSelfSignedCertsAreRejectedWhenNotInTheTrustStore() async -> Int { - let roots = CertificateStore([TestCertificate.ca1]) - - var verifier = Verifier(rootCertificates: roots) { RFC5280Policy(validationTime: TestCertificate.referenceTime) } - let result = await verifier.validate(leafCertificate: TestCertificate.isolatedSelfSignedCert, intermediates: CertificateStore([TestCertificate.intermediate1])) - - guard case .couldNotValidate(let policyResults) = result else { - fatalError("Incorrectly validated: \(result)") - } - return policyResults.count -} - -@available(macOS 11.0, *) -func testMissingRootFailsToBuild() async -> Int { - let roots = CertificateStore([]) - - var verifier = Verifier(rootCertificates: roots) { RFC5280Policy(validationTime: TestCertificate.referenceTime) } - let result = await verifier.validate(leafCertificate: TestCertificate.localhostLeaf, intermediates: CertificateStore([TestCertificate.intermediate1])) - - guard case .couldNotValidate(let policyResults) = result else { - fatalError("Accidentally validated: \(result)") - } - - return policyResults.count -} - -fileprivate struct FailIfCertInChainPolicy: VerifierPolicy { - let verifyingCriticalExtensions: [ASN1ObjectIdentifier] = [] - - private let forbiddenCert: Certificate - - init(forbiddenCert: Certificate) { - self.forbiddenCert = forbiddenCert - } - - func chainMeetsPolicyRequirements(chain: UnverifiedCertificateChain) async -> PolicyEvaluationResult { - if chain.contains(self.forbiddenCert) { - return .failsToMeetPolicy(reason: "chain must not contain \(self.forbiddenCert)") - } else { - return .meetsPolicy - } - } -} diff --git a/README.md b/README.md index b1e7219d..2089b106 100644 --- a/README.md +++ b/README.md @@ -48,3 +48,24 @@ dependencies: [ ``` For detailed usage and API documentation, check [the documentation](https://swiftpackageindex.com/apple/swift-certificates/main/documentation/x509). + +## Benchmarks + +Benchmarks for `swift-certificates` are in a separate Swift Package in the `Benchmarks` subfolder of this repository. +They use the [`package-benchmark`](https://github.com/ordo-one/package-benchmark) plugin. +Benchmarks depends on the [`jemalloc`](https://jemalloc.net) memory allocation library, which is used by `package-benchmark` to capture memory allocation statistics. +An installation guide can be found in the [Getting Started article](https://swiftpackageindex.com/ordo-one/package-benchmark/documentation/benchmark/gettingstarted#Installing-Prerequisites-and-Platform-Support) of `package-benchmark`. +Afterwards you can run the benchmarks from CLI by going to the `Benchmarks` subfolder (e.g. `cd Benchmarks`) and invoking: +``` +swift package benchmark +``` + +Profiling benchmarks or building the benchmarks in release mode in Xcode with `jemalloc` is currently not supported and requires disabling `jemalloc`. +Make sure Xcode is closed and then open it from the CLI with the `BENCHMARK_DISABLE_JEMALLOC=true` environment variable set e.g.: +``` +BENCHMARK_DISABLE_JEMALLOC=true xed . +``` + + +For more information please refer to `swift package benchmark --help` or the [documentation of `package-benchmark`](https://swiftpackageindex.com/ordo-one/package-benchmark/documentation/benchmark). + diff --git a/dev/alloc-limits-from-test-output b/dev/alloc-limits-from-test-output deleted file mode 100755 index 8708b9ec..00000000 --- a/dev/alloc-limits-from-test-output +++ /dev/null @@ -1,88 +0,0 @@ -#!/bin/bash -##===----------------------------------------------------------------------===## -## -## This source file is part of the SwiftCertificates open source project -## -## Copyright (c) 2023 Apple Inc. and the SwiftCertificates project authors -## Licensed under Apache License v2.0 -## -## See LICENSE.txt for license information -## See CONTRIBUTORS.txt for the list of SwiftCertificates project authors -## -## SPDX-License-Identifier: Apache-2.0 -## -##===----------------------------------------------------------------------===## - -# This script allows you to consume any Jenkins/alloc counter run output and -# convert it into the right for for the docker-compose script. - -set -eu - -mode_flag=${1---docker-compose} - -function usage() { - echo >&1 "Usage: $0 [--docker-compose|--export]" - echo >&1 - echo >&1 "Example:" - echo >&1 " # copy the output from the Jenkins CI into your clipboard, then" - echo >&1 " pbpaste | $0 --docker-compose" -} - -function die() { - echo >&2 "ERROR: $*" - exit 1 -} - -case "$mode_flag" in - --docker-compose) - mode=docker - ;; - --export) - mode=export - ;; - *) - usage - exit 1 - ;; -esac - -function allow_slack() { - raw="$1" - if [[ ! "$raw" =~ ^[0-9]+$ ]]; then - die "not a malloc count: '$raw'" - fi - if [[ "$raw" -lt 1000 ]]; then - echo "$raw" - return - fi - - allocs=$raw - while true; do - allocs=$(( allocs + 1 )) - if [[ "$allocs" =~ [0-9]+00$ || "$allocs" =~ [0-9]+50$ ]]; then - echo "$allocs" - return - fi - done -} - -grep -e "total number of mallocs" -e ".total_allocations" -e "export MAX_ALLOCS_ALLOWED_" | \ - sed -e "s/: total number of mallocs: /=/g" \ - -e "s/.total_allocations: /=/g" \ - -e "s/info: /test_/g" \ - -e "s/export MAX_ALLOCS_ALLOWED_/test_/g" | \ - grep -Eo 'test_[a-zA-Z0-9_-]+=[0-9]+' | sort | uniq | while read info; do - test_name=$(echo "$info" | sed "s/test_//g" | cut -d= -f1 ) - allocs=$(allow_slack "$(echo "$info" | cut -d= -f2 | sed "s/ //g")") - case "$mode" in - docker) - echo " - MAX_ALLOCS_ALLOWED_$test_name=$allocs" - ;; - export) - echo "export MAX_ALLOCS_ALLOWED_$test_name=$allocs" - ;; - *) - die "Unexpected mode: $mode" - ;; - esac -done diff --git a/dev/update-alloc-limits-to-last-completed-ci-build b/dev/update-alloc-limits-to-last-completed-ci-build deleted file mode 100755 index 6e381e1e..00000000 --- a/dev/update-alloc-limits-to-last-completed-ci-build +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -##===----------------------------------------------------------------------===## -## -## This source file is part of the SwiftCertificates open source project -## -## Copyright (c) 2023 Apple Inc. and the SwiftCertificates project authors -## Licensed under Apache License v2.0 -## -## See LICENSE.txt for license information -## See CONTRIBUTORS.txt for the list of SwiftCertificates project authors -## -## SPDX-License-Identifier: Apache-2.0 -## -##===----------------------------------------------------------------------===## - -set -eu -set -o pipefail - -here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -url_prefix=${1-"https://ci.swiftserver.group/job/swift-certificates-"} -target_repo=${2-"$here/.."} -tmpdir=$(mktemp -d /tmp/.last-build_XXXXXX) - -for f in 57 58 59 -nightly; do - echo "swift$f" - if [[ "$f" == "nightly" ]]; then - url="$url_prefix$f-prb/lastCompletedBuild/consoleFull" - else - url="${url_prefix}swift${f}-prb/lastCompletedBuild/consoleFull" - fi - echo "$url" - curl -s "$url" | "$here/alloc-limits-from-test-output" > "$tmpdir/limits$f" - - if [[ "$(wc -l < "$tmpdir/limits$f")" -lt 3 ]]; then - echo >&2 "ERROR: fewer than 3 limits found, something's not right" - exit 1 - fi - - docker_file=$(if [[ "$f" == "-nightly" ]]; then f=main; fi && ls "$target_repo/docker/docker-compose."*"$f"*".yaml") - - echo "$docker_file" - cat "$tmpdir/limits$f" - cat "$docker_file" | grep -v MAX_ALLOCS_ALLOWED | grep -B10000 "^ environment:" > "$tmpdir/pre$f" - cat "$docker_file" | grep -v MAX_ALLOCS_ALLOWED | grep -A10000 "^ environment:" | sed 1d > "$tmpdir/post$f" - cat "$tmpdir/pre$f" "$tmpdir/limits$f" "$tmpdir/post$f" > "$docker_file" -done - -rm -rf "$tmpdir" diff --git a/IntegrationTests/run-single-test.sh b/dev/update-benchmark-thresholds similarity index 66% rename from IntegrationTests/run-single-test.sh rename to dev/update-benchmark-thresholds index 42f0e0fa..37439e21 100755 --- a/IntegrationTests/run-single-test.sh +++ b/dev/update-benchmark-thresholds @@ -13,21 +13,16 @@ ## ##===----------------------------------------------------------------------===## -( -# this sub-shell is where the actual test is run set -eu -set -x set -o pipefail -test="$1" -tmp="$2" -root="$3" -g_show_info="$4" here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +target_repo=${2-"$here/.."} -source "$here/test_functions.sh" -source "$test" -wait -) -exit_code=$? -exit $exit_code +for f in 57 58 59 -nightly; do + echo "swift$f" + + docker_file=$(if [[ "$f" == "-nightly" ]]; then f=main; fi && ls "$target_repo/docker/docker-compose."*"$f"*".yaml") + + docker-compose -f docker/docker-compose.yaml -f $docker_file run update-benchmark-baseline +done diff --git a/docker/Dockerfile b/docker/Dockerfile index e4828b44..e5d34f80 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -21,3 +21,7 @@ ARG swiftformat_version=508.0.1 RUN git clone --branch $swiftformat_version --depth 1 https://github.com/apple/swift-format $HOME/.tools/swift-format-source RUN cd $HOME/.tools/swift-format-source && swift build -c release RUN ln -s $HOME/.tools/swift-format-source/.build/release/swift-format $HOME/.tools/swift-format + +# install jemalloc for running allocation benchmarks +RUN apt-get update +RUN apt-get install -y libjemalloc-dev diff --git a/docker/docker-compose.2204.57.yaml b/docker/docker-compose.2204.57.yaml index fde82f40..eed34280 100644 --- a/docker/docker-compose.2204.57.yaml +++ b/docker/docker-compose.2204.57.yaml @@ -12,12 +12,16 @@ services: test: image: swift-certificates:22.04-5.7 environment: - - MAX_ALLOCS_ALLOWED_parse_webpki_roots=200050 - - MAX_ALLOCS_ALLOWED_tiny_array_cow_append_contents_of=9050 - - MAX_ALLOCS_ALLOWED_tiny_array_non_allocating_operations=0 - - MAX_ALLOCS_ALLOWED_validation=120050 + - SWIFT_VERSION=5.7 - WARN_AS_ERROR_ARG=-Xswiftc -warnings-as-errors # - SANITIZER_ARG=--sanitize=thread # TSan broken still shell: image: swift-certificates:22.04-5.7 + environment: + - SWIFT_VERSION=5.7 + + update-benchmark-baseline: + image: swift-certificates:22.04-5.7 + environment: + - SWIFT_VERSION=5.7 diff --git a/docker/docker-compose.2204.58.yaml b/docker/docker-compose.2204.58.yaml index 49e0d7ef..c6ebebb2 100644 --- a/docker/docker-compose.2204.58.yaml +++ b/docker/docker-compose.2204.58.yaml @@ -12,13 +12,17 @@ services: test: image: swift-certificates:22.04-5.8 environment: - - MAX_ALLOCS_ALLOWED_parse_webpki_roots=200050 - - MAX_ALLOCS_ALLOWED_tiny_array_cow_append_contents_of=9050 - - MAX_ALLOCS_ALLOWED_tiny_array_non_allocating_operations=0 - - MAX_ALLOCS_ALLOWED_validation=120050 + - SWIFT_VERSION=5.8 - WARN_AS_ERROR_ARG=-Xswiftc -warnings-as-errors - IMPORT_CHECK_ARG=--explicit-target-dependency-import-check error # - SANITIZER_ARG=--sanitize=thread # TSan broken still shell: image: swift-certificates:22.04-5.8 + environment: + - SWIFT_VERSION=5.8 + + update-benchmark-baseline: + image: swift-certificates:22.04-5.8 + environment: + - SWIFT_VERSION=5.8 diff --git a/docker/docker-compose.2204.59.yaml b/docker/docker-compose.2204.59.yaml index fe6d68be..a5c9c179 100644 --- a/docker/docker-compose.2204.59.yaml +++ b/docker/docker-compose.2204.59.yaml @@ -11,13 +11,15 @@ services: test: image: swift-certificates:22.04-5.9 environment: - - MAX_ALLOCS_ALLOWED_parse_webpki_roots=200050 - - MAX_ALLOCS_ALLOWED_tiny_array_cow_append_contents_of=9050 - - MAX_ALLOCS_ALLOWED_tiny_array_non_allocating_operations=0 - - MAX_ALLOCS_ALLOWED_validation=120050 + - SWIFT_VERSION=5.9 - WARN_AS_ERROR_ARG=-Xswiftc -warnings-as-errors - IMPORT_CHECK_ARG=--explicit-target-dependency-import-check error # - SANITIZER_ARG=--sanitize=thread # TSan broken still shell: image: swift-certificates:22.04-5.9 + + update-benchmark-baseline: + image: swift-certificates:22.04-5.9 + environment: + - SWIFT_VERSION=5.9 diff --git a/docker/docker-compose.2204.main.yaml b/docker/docker-compose.2204.main.yaml index 02268684..946230b3 100644 --- a/docker/docker-compose.2204.main.yaml +++ b/docker/docker-compose.2204.main.yaml @@ -11,13 +11,15 @@ services: test: image: swift-certificates:22.04-main environment: - - MAX_ALLOCS_ALLOWED_parse_webpki_roots=200050 - - MAX_ALLOCS_ALLOWED_tiny_array_cow_append_contents_of=9050 - - MAX_ALLOCS_ALLOWED_tiny_array_non_allocating_operations=0 - - MAX_ALLOCS_ALLOWED_validation=120050 + - SWIFT_VERSION=main - WARN_AS_ERROR_ARG=-Xswiftc -warnings-as-errors - IMPORT_CHECK_ARG=--explicit-target-dependency-import-check error # - SANITIZER_ARG=--sanitize=thread # TSan broken still shell: image: swift-certificates:22.04-main + + update-benchmark-baseline: + image: swift-certificates:22.04-main + environment: + - SWIFT_VERSION=main diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 3193926a..31ead6d3 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -16,8 +16,8 @@ services: depends_on: [runtime-setup] volumes: - ~/.ssh:/root/.ssh - - ..:/code:z - working_dir: /code + - ..:/swift-certificates:z + working_dir: /swift-certificates soundness: <<: *common @@ -25,8 +25,11 @@ services: test: <<: *common - command: /bin/bash -xcl "swift $${SWIFT_TEST_VERB-test} $${WARN_AS_ERROR_ARG-} $${SANITIZER_ARG-} $${IMPORT_CHECK_ARG-} && ./scripts/integration_tests.sh $${INTEGRATION_TESTS_ARG-}" + command: /bin/bash -xcl "swift $${SWIFT_TEST_VERB-test} $${WARN_AS_ERROR_ARG-} $${SANITIZER_ARG-} $${IMPORT_CHECK_ARG-} && cd Benchmarks && swift package benchmark baseline check --check-absolute-path Thresholds/$${SWIFT_VERSION-}/" + update-benchmark-baseline: + <<: *common + command: /bin/bash -xcl "cd Benchmarks && swift package --scratch-path .build/$${SWIFT_VERSION-}/ --allow-writing-to-package-directory benchmark --format metricP90AbsoluteThresholds --path Thresholds/$${SWIFT_VERSION-}/" # util shell: diff --git a/scripts/integration_tests.sh b/scripts/integration_tests.sh deleted file mode 100755 index 5de4822d..00000000 --- a/scripts/integration_tests.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -##===----------------------------------------------------------------------===## -## -## This source file is part of the SwiftCertificates open source project -## -## Copyright (c) 2023 Apple Inc. and the SwiftCertificates project authors -## Licensed under Apache License v2.0 -## -## See LICENSE.txt for license information -## See CONTRIBUTORS.txt for the list of SwiftCertificates project authors -## -## SPDX-License-Identifier: Apache-2.0 -## -##===----------------------------------------------------------------------===## - -set +ex - -mkdir -p .build # for the junit.xml file -./IntegrationTests/run-tests.sh --junit-xml .build/junit-sh-tests.xml -i $@ diff --git a/scripts/run-swift-format.sh b/scripts/run-swift-format.sh index 13d766b5..1e9c2ce1 100644 --- a/scripts/run-swift-format.sh +++ b/scripts/run-swift-format.sh @@ -39,6 +39,7 @@ swiftformat_bin=${swiftformat_bin:-$(command -v swift-format)} || fatal "❌ swi "${swiftformat_bin}" lint \ --parallel --recursive --strict \ "${repo_root}/Sources" "${repo_root}/Tests" \ + "${repo_root}/Benchmarks/Benchmarks" \ && swift_format_rc=$? || swift_format_rc=$? if [[ "${swift_format_rc}" -ne 0 ]]; then @@ -46,7 +47,7 @@ if [[ "${swift_format_rc}" -ne 0 ]]; then To fix, run the following command: - % swift-format format --parallel --recursive --in-place Sources Tests + % swift-format format --parallel --recursive --in-place Sources Tests Benchmarks/Benchmarks " exit "${swift_format_rc}" fi