diff --git a/app/lib/frontend/handlers/custom_api.dart b/app/lib/frontend/handlers/custom_api.dart index 4e3d75347..0141b1821 100644 --- a/app/lib/frontend/handlers/custom_api.dart +++ b/app/lib/frontend/handlers/custom_api.dart @@ -21,7 +21,6 @@ import '../../scorecard/backend.dart'; import '../../search/backend.dart'; import '../../search/search_client.dart'; import '../../search/search_service.dart'; -import '../../service/download_counts/backend.dart'; import '../../service/topics/count_topics.dart'; import '../../shared/configuration.dart'; import '../../shared/exceptions.dart'; @@ -218,38 +217,7 @@ Future packageVersionScoreHandler( {String? version}) async { checkPackageVersionParams(package, version); return (await cache.versionScore(package, version).get(() async { - final pkg = await packageBackend.lookupPackage(package); - if (pkg == null) { - throw NotFoundException.resource('package "$package"'); - } - final v = - (version == null || version == 'latest') ? pkg.latestVersion! : version; - final pv = await packageBackend.lookupPackageVersion(package, v); - if (pv == null) { - throw NotFoundException.resource('package "$package" version "$version"'); - } - - var updated = pkg.updated; - final card = await scoreCardBackend.getScoreCardData(package, v); - if (updated == null || card.updated?.isAfter(updated) == true) { - updated = card.updated; - } - - final tags = { - ...pkg.getTags(), - ...pv.getTags(), - ...?card.derivedTags, - }; - - return VersionScore( - grantedPoints: card.grantedPubPoints, - maxPoints: card.maxPubPoints, - likeCount: pkg.likes, - downloadCount30Days: - downloadCountsBackend.lookup30DaysTotalCounts(package), - tags: tags.toList(), - lastUpdated: updated, - ); + return await scoreCardBackend.getVersionScore(package, version: version); }))!; } @@ -475,15 +443,7 @@ Future getPackageOptionsHandler( shelf.Request request, String package, ) async { - checkPackageVersionParams(package); - final p = await packageBackend.lookupPackage(package); - if (p == null) { - throw NotFoundException.resource(package); - } - return PkgOptions( - isDiscontinued: p.isDiscontinued, - isUnlisted: p.isUnlisted, - ); + return await packageBackend.getPackageOptions(package); } /// Handles `PUT /api/packages//options`. diff --git a/app/lib/package/api_export/api_exporter.dart b/app/lib/package/api_export/api_exporter.dart index 4215cd963..99f3b45dc 100644 --- a/app/lib/package/api_export/api_exporter.dart +++ b/app/lib/package/api_export/api_exporter.dart @@ -10,6 +10,7 @@ import 'package:gcloud/service_scope.dart' as ss; import 'package:gcloud/storage.dart'; import 'package:logging/logging.dart'; import 'package:pub_dev/frontend/handlers/atom_feed.dart'; +import 'package:pub_dev/scorecard/backend.dart'; import 'package:pub_dev/service/security_advisories/backend.dart'; import 'package:pub_dev/shared/exceptions.dart'; import 'package:pub_dev/shared/parallel_foreach.dart'; @@ -244,6 +245,22 @@ final class ApiExporter { versionListing, forceWrite: forceWrite, ); + await _api.package(package).likes.write( + await packageBackend.getPackageLikesCount(package), + forceWrite: forceWrite, + ); + await _api.package(package).options.write( + await packageBackend.getPackageOptions(package), + forceWrite: forceWrite, + ); + await _api.package(package).publisher.write( + await packageBackend.getPublisherInfo(package), + forceWrite: forceWrite, + ); + await _api.package(package).score.write( + await scoreCardBackend.getVersionScore(package), + forceWrite: forceWrite, + ); await _api.package(package).feedAtomFile.write( await buildPackageAtomFeedContent(package), forceWrite: forceWrite, diff --git a/app/lib/package/api_export/exported_api.dart b/app/lib/package/api_export/exported_api.dart index 92b61fc52..1ca46b136 100644 --- a/app/lib/package/api_export/exported_api.dart +++ b/app/lib/package/api_export/exported_api.dart @@ -6,6 +6,7 @@ import 'dart:async'; import 'dart:convert'; import 'dart:io'; +import 'package:_pub_shared/data/account_api.dart'; import 'package:_pub_shared/data/advisories_api.dart'; import 'package:_pub_shared/data/package_api.dart'; import 'package:clock/clock.dart'; @@ -273,6 +274,20 @@ final class ExportedPackage { ExportedJsonFile get advisories => _suffix('/advisories'); + /// Interface for writing `/api/packages//likes`. + ExportedJsonFile get likes => + _suffix('/likes'); + + /// Interface for writing `/api/packages//options`. + ExportedJsonFile get options => _suffix('/options'); + + /// Interface for writing `/api/packages//publisher`. + ExportedJsonFile get publisher => + _suffix('/publisher'); + + /// Interface for writing `/api/packages//score`. + ExportedJsonFile get score => _suffix('/score'); + /// Interface for writing `/api/packages//feed.atom` ExportedAtomFeedFile get feedAtomFile => ExportedAtomFeedFile._( _owner, @@ -406,6 +421,10 @@ final class ExportedPackage { await Future.wait([ _owner._pool.withResource(() async => await versions.delete()), _owner._pool.withResource(() async => await advisories.delete()), + _owner._pool.withResource(() async => await likes.delete()), + _owner._pool.withResource(() async => await options.delete()), + _owner._pool.withResource(() async => await publisher.delete()), + _owner._pool.withResource(() async => await score.delete()), _owner._pool.withResource(() async => await feedAtomFile.delete()), ..._owner._prefixes.map((prefix) async { await _owner._listBucket( diff --git a/app/lib/package/backend.dart b/app/lib/package/backend.dart index 21eec1d71..3d81e0bbe 100644 --- a/app/lib/package/backend.dart +++ b/app/lib/package/backend.dart @@ -18,6 +18,7 @@ import 'package:meta/meta.dart'; import 'package:pool/pool.dart'; import 'package:pub_dev/package/api_export/api_exporter.dart'; import 'package:pub_dev/package/tarball_storage.dart'; +import 'package:pub_dev/scorecard/backend.dart'; import 'package:pub_dev/service/async_queue/async_queue.dart'; import 'package:pub_dev/service/rate_limit/rate_limit.dart'; import 'package:pub_dev/shared/versions.dart'; @@ -419,6 +420,20 @@ class PackageBackend { return count; } + /// Returns the package options. + Future getPackageOptions(String package) async { + checkPackageVersionParams(package); + final p = await packageBackend.lookupPackage(package); + if (p == null) { + throw NotFoundException.resource(package); + } + return api.PkgOptions( + isDiscontinued: p.isDiscontinued, + replacedBy: p.replacedBy, + isUnlisted: p.isUnlisted, + ); + } + /// Updates [options] on [package]. Future updateOptions(String package, api.PkgOptions options) async { final authenticatedUser = await requireAuthenticatedWebUser(); @@ -479,6 +494,7 @@ class PackageBackend { }); await purgePackageCache(package); await taskBackend.trackPackage(package); + await apiExporter!.synchronizePackage(package); } /// Updates [options] on [package]/[version], assuming the current user @@ -517,6 +533,9 @@ class PackageBackend { } }); await purgePackageCache(package); + await purgeScorecardData(package, version, + isLatest: pkg.latestVersion == version); + await apiExporter!.synchronizePackage(package); } /// Verifies an update to the credential-less publishing settings and @@ -781,7 +800,7 @@ class PackageBackend { if (currentPublisherId != null) { await purgePublisherCache(publisherId: currentPublisherId); } - + await apiExporter!.synchronizePackage(packageName); return rs; } diff --git a/app/lib/scorecard/backend.dart b/app/lib/scorecard/backend.dart index e7e03c00c..2dace75a8 100644 --- a/app/lib/scorecard/backend.dart +++ b/app/lib/scorecard/backend.dart @@ -4,11 +4,13 @@ import 'dart:async'; +import 'package:_pub_shared/data/package_api.dart'; import 'package:clock/clock.dart'; import 'package:gcloud/service_scope.dart' as ss; import 'package:logging/logging.dart'; import 'package:meta/meta.dart'; import 'package:pool/pool.dart'; +import 'package:pub_dev/service/download_counts/backend.dart'; import 'package:pub_dev/service/download_counts/computations.dart'; import 'package:pub_dev/shared/exceptions.dart'; import 'package:pub_dev/task/backend.dart'; @@ -186,6 +188,43 @@ class ScoreCardBackend { final pv = list[1] as PackageVersion?; return PackageStatus.fromModels(p, pv); } + + /// Return the version score object served in the API. + Future getVersionScore(String package, + {String? version}) async { + final pkg = await packageBackend.lookupPackage(package); + if (pkg == null) { + throw NotFoundException.resource('package "$package"'); + } + final v = + (version == null || version == 'latest') ? pkg.latestVersion! : version; + final pv = await packageBackend.lookupPackageVersion(package, v); + if (pv == null) { + throw NotFoundException.resource('package "$package" version "$version"'); + } + + var updated = pkg.updated; + final card = await scoreCardBackend.getScoreCardData(package, v); + if (updated == null || card.updated?.isAfter(updated) == true) { + updated = card.updated; + } + + final tags = { + ...pkg.getTags(), + ...pv.getTags(), + ...?card.derivedTags, + }; + + return VersionScore( + grantedPoints: card.grantedPubPoints, + maxPoints: card.maxPubPoints, + likeCount: pkg.likes, + downloadCount30Days: + downloadCountsBackend.lookup30DaysTotalCounts(package), + tags: tags.toList(), + lastUpdated: updated, + ); + } } Future purgeScorecardData( diff --git a/app/lib/task/backend.dart b/app/lib/task/backend.dart index 33657f507..faf62643d 100644 --- a/app/lib/task/backend.dart +++ b/app/lib/task/backend.dart @@ -18,6 +18,7 @@ import 'package:indexed_blob/indexed_blob.dart' show BlobIndex, FileRange; import 'package:logging/logging.dart' show Logger; import 'package:pana/models.dart' show Summary; import 'package:pool/pool.dart' show Pool; +import 'package:pub_dev/package/api_export/api_exporter.dart'; import 'package:pub_dev/package/backend.dart'; import 'package:pub_dev/package/models.dart'; import 'package:pub_dev/package/upload_signer_service.dart'; @@ -721,6 +722,7 @@ class TaskBackend { // Clearing the state cache after the update. await _purgeCache(package, version); + await apiExporter!.synchronizePackage(package); // If nothing else is running on the instance, delete it! // We do this in a microtask after returning, so that it doesn't slow down diff --git a/app/test/admin/exported_api_sync_test.dart b/app/test/admin/exported_api_sync_test.dart index e8028ca5e..7f240b27a 100644 --- a/app/test/admin/exported_api_sync_test.dart +++ b/app/test/admin/exported_api_sync_test.dart @@ -71,8 +71,8 @@ void main() { testWithProfile('baseline checks', fn: () async { await syncExportedApi(); final data = await listExportedApi(); - expect(data.keys, hasLength(greaterThan(20))); - expect(data.keys, hasLength(lessThan(40))); + expect(data.keys, hasLength(greaterThan(50))); + expect(data.keys, hasLength(lessThan(70))); final oxygenFiles = data.keys.where((k) => k.contains('oxygen')).toSet(); expect(oxygenFiles, hasLength(greaterThan(5))); @@ -82,12 +82,20 @@ void main() { '$runtimeVersion/api/archives/oxygen-2.0.0-dev.tar.gz', '$runtimeVersion/api/packages/oxygen', '$runtimeVersion/api/packages/oxygen/advisories', + '$runtimeVersion/api/packages/oxygen/likes', + '$runtimeVersion/api/packages/oxygen/options', + '$runtimeVersion/api/packages/oxygen/publisher', + '$runtimeVersion/api/packages/oxygen/score', '$runtimeVersion/api/packages/oxygen/feed.atom', 'latest/api/archives/oxygen-1.0.0.tar.gz', 'latest/api/archives/oxygen-1.2.0.tar.gz', 'latest/api/archives/oxygen-2.0.0-dev.tar.gz', 'latest/api/packages/oxygen', 'latest/api/packages/oxygen/advisories', + 'latest/api/packages/oxygen/likes', + 'latest/api/packages/oxygen/options', + 'latest/api/packages/oxygen/publisher', + 'latest/api/packages/oxygen/score', 'latest/api/packages/oxygen/feed.atom', }); diff --git a/app/test/package/api_export/api_exporter_test.dart b/app/test/package/api_export/api_exporter_test.dart index 76b0c0926..d8a89c0d9 100644 --- a/app/test/package/api_export/api_exporter_test.dart +++ b/app/test/package/api_export/api_exporter_test.dart @@ -132,8 +132,36 @@ Future _testExportedApiSynchronization( isNotNull, ); expect( - await bucket.readBytes('$runtimeVersion/api/packages/foo/feed.atom'), - isNotNull, + await bucket.readGzippedJson('$runtimeVersion/api/packages/foo/likes'), + { + 'package': 'foo', + 'likes': 0, + }, + ); + expect( + await bucket.readGzippedJson('$runtimeVersion/api/packages/foo/options'), + { + 'isDiscontinued': false, + 'replacedBy': null, + 'isUnlisted': false, + }, + ); + expect( + await bucket + .readGzippedJson('$runtimeVersion/api/packages/foo/publisher'), + { + 'publisherId': null, + }, + ); + expect( + await bucket.readGzippedJson('$runtimeVersion/api/packages/foo/score'), + { + 'grantedPoints': isNotNull, + 'maxPoints': isNotNull, + 'likeCount': isNotNull, + 'tags': isNotEmpty, + 'lastUpdated': isNotNull, + }, ); expect( await bucket.readGzippedJson('$runtimeVersion/api/packages/foo'), @@ -143,6 +171,10 @@ Future _testExportedApiSynchronization( 'versions': hasLength(1), }, ); + expect( + await bucket.readString('$runtimeVersion/api/packages/foo/feed.atom'), + contains('v1.0.0 of foo'), + ); expect( await bucket .readGzippedJson('$runtimeVersion/api/package-name-completion-data'), @@ -156,10 +188,6 @@ Future _testExportedApiSynchronization( await bucket.readString('$runtimeVersion/feed.atom'), contains('v1.0.0 of foo'), ); - expect( - await bucket.readString('$runtimeVersion/api/packages/foo/feed.atom'), - contains('v1.0.0 of foo'), - ); } _log.info('## New package'); @@ -197,6 +225,22 @@ Future _testExportedApiSynchronization( await bucket.readString('latest/api/packages/foo/feed.atom'), contains('v1.0.0 of foo'), ); + expect( + await bucket.readGzippedJson('latest/api/packages/foo/likes'), + isNotNull, + ); + expect( + await bucket.readGzippedJson('latest/api/packages/foo/options'), + isNotNull, + ); + expect( + await bucket.readGzippedJson('latest/api/packages/foo/publisher'), + isNotNull, + ); + expect( + await bucket.readGzippedJson('latest/api/packages/foo/score'), + isNotNull, + ); // Note. that name completion data won't be updated until search caches // are purged, so we won't test that it is updated. @@ -444,6 +488,10 @@ Future _testExportedApiSynchronization( await bucket.readGzippedJson('latest/api/packages/bar'), isNull, ); + expect( + await bucket.readGzippedJson('latest/api/packages/bar/options'), + isNull, + ); expect( await bucket.readGzippedJson('latest/api/packages/feed.atom'), isNull, @@ -484,6 +532,10 @@ Future _testExportedApiSynchronization( 'versions': hasLength(2), }, ); + expect( + await bucket.readGzippedJson('latest/api/packages/bar/options'), + isNotNull, + ); expect( await bucket.readBytes('latest/api/archives/bar-2.0.0.tar.gz'), isNotNull, diff --git a/app/test/task/task_test.dart b/app/test/task/task_test.dart index 19aedd253..3220f5d9b 100644 --- a/app/test/task/task_test.dart +++ b/app/test/task/task_test.dart @@ -190,7 +190,9 @@ void main() { await fakeTime.elapse(minutes: 10); }); - testWithFakeTime('failing instances will be retried', (fakeTime) async { + testWithFakeTime('failing instances will be retried', expectedLogMessages: [ + 'SHOUT [pub-notice:cached_value] Updating cached `thirtyDaysTotalDownloadCounts` value failed.', + ], (fakeTime) async { await taskBackend.backfillTrackingState(); await fakeTime.elapse(minutes: 1);