diff --git a/app/bin/service/analyzer.dart b/app/bin/service/analyzer.dart index 7a8717ab12..106251a3d3 100644 --- a/app/bin/service/analyzer.dart +++ b/app/bin/service/analyzer.dart @@ -7,15 +7,20 @@ import 'dart:isolate'; import 'package:appengine/appengine.dart'; import 'package:gcloud/db.dart' as db; +import 'package:gcloud/storage.dart'; import 'package:logging/logging.dart'; import 'package:pub_dartlang_org/history/backend.dart'; import 'package:pub_dartlang_org/job/backend.dart'; import 'package:pub_dartlang_org/job/job.dart'; +import 'package:pub_dartlang_org/scorecard/backend.dart'; +import 'package:pub_dartlang_org/scorecard/scorecard_memcache.dart'; import 'package:pub_dartlang_org/shared/analyzer_memcache.dart'; +import 'package:pub_dartlang_org/shared/configuration.dart'; import 'package:pub_dartlang_org/shared/dartdoc_client.dart'; import 'package:pub_dartlang_org/shared/dartdoc_memcache.dart'; import 'package:pub_dartlang_org/shared/handler_helpers.dart'; +import 'package:pub_dartlang_org/shared/popularity_storage.dart'; import 'package:pub_dartlang_org/shared/scheduler_stats.dart'; import 'package:pub_dartlang_org/shared/service_utils.dart'; @@ -48,7 +53,7 @@ Future _frontendMain(FrontendEntryMessage message) async { )); await withAppEngineServices(() async { - _registerServices(); + await _registerServices(); await runHandler(logger, analyzerServiceHandler); }); } @@ -59,7 +64,7 @@ Future _workerMain(WorkerEntryMessage message) async { message.protocolSendPort.send(new WorkerProtocolMessage()); await withAppEngineServices(() async { - _registerServices(); + await _registerServices(); final jobProcessor = new AnalyzerJobProcessor(); final jobMaintenance = new JobMaintenance(db.dbService, jobProcessor); @@ -71,11 +76,18 @@ Future _workerMain(WorkerEntryMessage message) async { }); } -void _registerServices() { +Future _registerServices() async { + final Bucket popularityBucket = + storageService.bucket(activeConfiguration.popularityDumpBucketName); + registerPopularityStorage( + new PopularityStorage(storageService, popularityBucket)); + await popularityStorage.init(); registerAnalysisBackend(new AnalysisBackend(db.dbService)); registerAnalyzerMemcache(new AnalyzerMemcache(memcacheService)); registerDartdocMemcache(new DartdocMemcache(memcacheService)); registerDartdocClient(new DartdocClient()); registerHistoryBackend(new HistoryBackend(db.dbService)); registerJobBackend(new JobBackend(db.dbService)); + registerScoreCardMemcache(new ScoreCardMemcache(memcacheService)); + registerScoreCardBackend(new ScoreCardBackend(db.dbService)); } diff --git a/app/bin/service/dartdoc.dart b/app/bin/service/dartdoc.dart index 3412dfa7c3..76313751cd 100644 --- a/app/bin/service/dartdoc.dart +++ b/app/bin/service/dartdoc.dart @@ -20,6 +20,7 @@ import 'package:pub_dartlang_org/shared/analyzer_client.dart'; import 'package:pub_dartlang_org/shared/configuration.dart'; import 'package:pub_dartlang_org/shared/dartdoc_memcache.dart'; import 'package:pub_dartlang_org/shared/handler_helpers.dart'; +import 'package:pub_dartlang_org/shared/popularity_storage.dart'; import 'package:pub_dartlang_org/shared/scheduler_stats.dart'; import 'package:pub_dartlang_org/shared/service_utils.dart'; @@ -81,6 +82,12 @@ Future _workerMain(WorkerEntryMessage message) async { } Future _registerServices() async { + final Bucket popularityBucket = + storageService.bucket(activeConfiguration.popularityDumpBucketName); + registerPopularityStorage( + new PopularityStorage(storageService, popularityBucket)); + await popularityStorage.init(); + registerDartdocMemcache(new DartdocMemcache(memcacheService)); registerAnalysisBackend(new AnalysisBackend(dbService)); diff --git a/app/lib/analyzer/pana_runner.dart b/app/lib/analyzer/pana_runner.dart index 0fbb13a100..631ead0024 100644 --- a/app/lib/analyzer/pana_runner.dart +++ b/app/lib/analyzer/pana_runner.dart @@ -11,7 +11,10 @@ import 'package:pana/src/maintenance.dart'; import 'package:pana/src/version.dart' as pana_version; import '../job/job.dart'; -import '../shared/analyzer_client.dart' show createPanaSummaryForLegacy; +import '../scorecard/backend.dart'; +import '../scorecard/models.dart'; +import '../shared/analyzer_client.dart' + show createPanaSummaryForLegacy, getAllSuggestions; import '../shared/analyzer_service.dart'; import '../shared/dartdoc_client.dart'; import '../shared/packages_overrides.dart'; @@ -62,6 +65,7 @@ class AnalyzerJobProcessor extends JobProcessor { analysis.analysisStatus = AnalysisStatus.discontinued; analysis.maintenanceScore = 0.0; await analysisBackend.storeAnalysis(analysis); + await _storeScoreCard(job, packageStatus, null); return JobStatus.skipped; } @@ -71,17 +75,19 @@ class AnalyzerJobProcessor extends JobProcessor { analysis.analysisStatus = AnalysisStatus.outdated; analysis.maintenanceScore = 0.0; await analysisBackend.storeAnalysis(analysis); + await _storeScoreCard(job, packageStatus, null); return JobStatus.skipped; } if (packageStatus.isLegacy) { _logger.info('Package is on legacy SDK: $job.'); analysis.analysisStatus = AnalysisStatus.legacy; - analysis.analysisJson = - createPanaSummaryForLegacy(job.packageName, job.packageVersion) - .toJson(); + final summary = + createPanaSummaryForLegacy(job.packageName, job.packageVersion); + analysis.analysisJson = summary.toJson(); analysis.maintenanceScore = 0.0; await analysisBackend.storeAnalysis(analysis); + await _storeScoreCard(job, packageStatus, summary); return JobStatus.skipped; } @@ -124,10 +130,12 @@ class AnalyzerJobProcessor extends JobProcessor { } JobStatus status = JobStatus.failed; + Summary scoreCardSummary = summary; if (summary == null) { analysis.analysisStatus = AnalysisStatus.aborted; } else { summary = applyPlatformOverride(summary); + scoreCardSummary = summary; summary = await _expandSummary(summary, packageStatus.age); final bool lastRunWithErrors = summary.suggestions?.where((s) => s.isError)?.isNotEmpty ?? false; @@ -143,6 +151,7 @@ class AnalyzerJobProcessor extends JobProcessor { } final backendStatus = await analysisBackend.storeAnalysis(analysis); + await _storeScoreCard(job, packageStatus, scoreCardSummary); if (backendStatus.isLatestStable && analysis.analysisStatus != AnalysisStatus.success && @@ -182,4 +191,23 @@ class AnalyzerJobProcessor extends JobProcessor { } return summary; } + + Future _storeScoreCard(Job job, PackageStatus status, Summary summary) async { + final reportStatus = + summary == null ? ReportStatus.aborted : ReportStatus.success; + await scoreCardBackend.updateReport( + job.packageName, + job.packageVersion, + new PanaReport( + reportStatus: reportStatus, + healthScore: summary?.health?.healthScore ?? 0.0, + maintenanceScore: + summary == null ? 0.0 : getMaintenanceScore(summary.maintenance), + platformTags: indexDartPlatform(summary?.platform), + platformReason: summary?.platform?.reason, + pkgDependencies: summary?.pkgResolution?.dependencies, + suggestions: getAllSuggestions(summary), + )); + await scoreCardBackend.updateScoreCard(job.packageName, job.packageVersion); + } } diff --git a/app/lib/scorecard/backend.dart b/app/lib/scorecard/backend.dart index fbb4fd9ce8..bc5a3710a9 100644 --- a/app/lib/scorecard/backend.dart +++ b/app/lib/scorecard/backend.dart @@ -142,8 +142,8 @@ class ScoreCardBackend { Future updateScoreCard(String packageName, String packageVersion) async { final key = scoreCardKey(packageName, packageVersion); final pAndPv = await _db.lookup([key.parent, key.parent.parent]); - final package = pAndPv[0] as Package; - final version = pAndPv[1] as PackageVersion; + final version = pAndPv[0] as PackageVersion; + final package = pAndPv[1] as Package; if (package == null || version == null) { throw new Exception('Unable to lookup $packageName $packageVersion.'); } @@ -168,7 +168,7 @@ class ScoreCardBackend { scoreCard.updated = new DateTime.now().toUtc(); } - scoreCard.flags = null; + scoreCard.flags.clear(); if (package.isDiscontinued ?? false) { scoreCard.addFlag(PackageFlags.isDiscontinued); } diff --git a/app/lib/scorecard/helpers.dart b/app/lib/scorecard/helpers.dart index 0a0443e14d..641657f914 100644 --- a/app/lib/scorecard/helpers.dart +++ b/app/lib/scorecard/helpers.dart @@ -7,6 +7,8 @@ import 'package:gcloud/db.dart' as db; import '../frontend/models.dart' show Package, PackageVersion; import '../shared/versions.dart' as versions; +import 'models.dart'; + db.Key scoreCardKey( String packageName, String packageVersion, { @@ -15,5 +17,6 @@ db.Key scoreCardKey( runtimeVersion ??= versions.runtimeVersion; return db.dbService.emptyKey .append(Package, id: packageName) - .append(PackageVersion, id: packageVersion); + .append(PackageVersion, id: packageVersion) + .append(ScoreCard, id: runtimeVersion); } diff --git a/app/lib/scorecard/models.dart b/app/lib/scorecard/models.dart index cedf43ae15..ed834107bb 100644 --- a/app/lib/scorecard/models.dart +++ b/app/lib/scorecard/models.dart @@ -80,16 +80,17 @@ class ScoreCard extends db.ExpandoModel { /// The platform tags (flutter, web, other). @CompatibleStringListProperty() - List platformTags; + List platformTags = []; /// The flags for the package, version or analysis. /// Example values: entries from [PackageFlags]. @CompatibleStringListProperty() - List flags; + List flags = []; /// The report types that are already done for the ScoreCard. + /// Contains values from [ReportType]. @CompatibleStringListProperty() - List reportTypes; + List reportTypes = []; ScoreCard(); @@ -141,7 +142,7 @@ class ScoreCard extends db.ExpandoModel { healthScore = (panaReport?.healthScore ?? 0.0) * (0.9 + ((dartdocReport?.coverageScore ?? 1.0) * 0.1)); maintenanceScore = panaReport?.maintenanceScore ?? 0.0; - platformTags = panaReport?.platformTags; + platformTags = panaReport?.platformTags ?? []; reportTypes = [ panaReport == null ? null : ReportType.pana, dartdocReport == null ? null : ReportType.dartdoc, diff --git a/app/lib/shared/analyzer_client.dart b/app/lib/shared/analyzer_client.dart index bec489225e..6ae7f9194d 100644 --- a/app/lib/shared/analyzer_client.dart +++ b/app/lib/shared/analyzer_client.dart @@ -243,24 +243,27 @@ class AnalysisView { return _summary?.health?.healthScore ?? 0.0; } - List get suggestions { - final list = []; - if (_summary?.suggestions != null) { - list.addAll(_summary.suggestions); - } - if (_summary?.health?.suggestions != null) { - list.addAll(_summary.health.suggestions); - } - if (_summary?.maintenance?.suggestions != null) { - list.addAll(_summary.maintenance.suggestions); - } - list.sort(); - return list; - } + List get suggestions => getAllSuggestions(_summary); double get maintenanceScore => _data?.maintenanceScore ?? 0.0; } +List getAllSuggestions(Summary summary) { + if (summary == null) return null; + final list = []; + if (summary.suggestions != null) { + list.addAll(summary.suggestions); + } + if (summary.health?.suggestions != null) { + list.addAll(summary.health.suggestions); + } + if (summary.maintenance?.suggestions != null) { + list.addAll(summary.maintenance.suggestions); + } + list.sort(); + return list; +} + Summary createPanaSummaryForLegacy(String packageName, String packageVersion) { return new Summary( runtimeInfo: new PanaRuntimeInfo(),