Skip to content

Export more package API endpoints. #8758

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 2 additions & 42 deletions app/lib/frontend/handlers/custom_api.dart
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ import '../../scorecard/backend.dart';
import '../../search/backend.dart';
import '../../search/search_client.dart';
import '../../search/search_service.dart';
import '../../service/download_counts/backend.dart';
import '../../service/topics/count_topics.dart';
import '../../shared/configuration.dart';
import '../../shared/exceptions.dart';
Expand Down Expand Up @@ -218,38 +217,7 @@ Future<VersionScore> packageVersionScoreHandler(
{String? version}) async {
checkPackageVersionParams(package, version);
return (await cache.versionScore(package, version).get(() async {
final pkg = await packageBackend.lookupPackage(package);
if (pkg == null) {
throw NotFoundException.resource('package "$package"');
}
final v =
(version == null || version == 'latest') ? pkg.latestVersion! : version;
final pv = await packageBackend.lookupPackageVersion(package, v);
if (pv == null) {
throw NotFoundException.resource('package "$package" version "$version"');
}

var updated = pkg.updated;
final card = await scoreCardBackend.getScoreCardData(package, v);
if (updated == null || card.updated?.isAfter(updated) == true) {
updated = card.updated;
}

final tags = <String>{
...pkg.getTags(),
...pv.getTags(),
...?card.derivedTags,
};

return VersionScore(
grantedPoints: card.grantedPubPoints,
maxPoints: card.maxPubPoints,
likeCount: pkg.likes,
downloadCount30Days:
downloadCountsBackend.lookup30DaysTotalCounts(package),
tags: tags.toList(),
lastUpdated: updated,
);
return await scoreCardBackend.getVersionScore(package, version: version);
}))!;
}

Expand Down Expand Up @@ -475,15 +443,7 @@ Future<PkgOptions> getPackageOptionsHandler(
shelf.Request request,
String package,
) async {
checkPackageVersionParams(package);
final p = await packageBackend.lookupPackage(package);
if (p == null) {
throw NotFoundException.resource(package);
}
return PkgOptions(
isDiscontinued: p.isDiscontinued,
isUnlisted: p.isUnlisted,
);
return await packageBackend.getPackageOptions(package);
}

/// Handles `PUT /api/packages/<package>/options`.
Expand Down
17 changes: 17 additions & 0 deletions app/lib/package/api_export/api_exporter.dart
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import 'package:gcloud/service_scope.dart' as ss;
import 'package:gcloud/storage.dart';
import 'package:logging/logging.dart';
import 'package:pub_dev/frontend/handlers/atom_feed.dart';
import 'package:pub_dev/scorecard/backend.dart';
import 'package:pub_dev/service/security_advisories/backend.dart';
import 'package:pub_dev/shared/exceptions.dart';
import 'package:pub_dev/shared/parallel_foreach.dart';
Expand Down Expand Up @@ -244,6 +245,22 @@ final class ApiExporter {
versionListing,
forceWrite: forceWrite,
);
await _api.package(package).likes.write(
await packageBackend.getPackageLikesCount(package),
forceWrite: forceWrite,
);
await _api.package(package).options.write(
await packageBackend.getPackageOptions(package),
forceWrite: forceWrite,
);
await _api.package(package).publisher.write(
await packageBackend.getPublisherInfo(package),
forceWrite: forceWrite,
);
await _api.package(package).score.write(
await scoreCardBackend.getVersionScore(package),
forceWrite: forceWrite,
);
await _api.package(package).feedAtomFile.write(
await buildPackageAtomFeedContent(package),
forceWrite: forceWrite,
Expand Down
19 changes: 19 additions & 0 deletions app/lib/package/api_export/exported_api.dart
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import 'dart:async';
import 'dart:convert';
import 'dart:io';

import 'package:_pub_shared/data/account_api.dart';
import 'package:_pub_shared/data/advisories_api.dart';
import 'package:_pub_shared/data/package_api.dart';
import 'package:clock/clock.dart';
Expand Down Expand Up @@ -273,6 +274,20 @@ final class ExportedPackage {
ExportedJsonFile<ListAdvisoriesResponse> get advisories =>
_suffix<ListAdvisoriesResponse>('/advisories');

/// Interface for writing `/api/packages/<package>/likes`.
ExportedJsonFile<PackageLikesCount> get likes =>
_suffix<PackageLikesCount>('/likes');

/// Interface for writing `/api/packages/<package>/options`.
ExportedJsonFile<PkgOptions> get options => _suffix<PkgOptions>('/options');

/// Interface for writing `/api/packages/<package>/publisher`.
ExportedJsonFile<PackagePublisherInfo> get publisher =>
_suffix<PackagePublisherInfo>('/publisher');

/// Interface for writing `/api/packages/<package>/score`.
ExportedJsonFile<VersionScore> get score => _suffix<VersionScore>('/score');

/// Interface for writing `/api/packages/<package>/feed.atom`
ExportedAtomFeedFile get feedAtomFile => ExportedAtomFeedFile._(
_owner,
Expand Down Expand Up @@ -406,6 +421,10 @@ final class ExportedPackage {
await Future.wait([
_owner._pool.withResource(() async => await versions.delete()),
_owner._pool.withResource(() async => await advisories.delete()),
_owner._pool.withResource(() async => await likes.delete()),
_owner._pool.withResource(() async => await options.delete()),
_owner._pool.withResource(() async => await publisher.delete()),
_owner._pool.withResource(() async => await score.delete()),
_owner._pool.withResource(() async => await feedAtomFile.delete()),
..._owner._prefixes.map((prefix) async {
await _owner._listBucket(
Expand Down
21 changes: 20 additions & 1 deletion app/lib/package/backend.dart
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import 'package:meta/meta.dart';
import 'package:pool/pool.dart';
import 'package:pub_dev/package/api_export/api_exporter.dart';
import 'package:pub_dev/package/tarball_storage.dart';
import 'package:pub_dev/scorecard/backend.dart';
import 'package:pub_dev/service/async_queue/async_queue.dart';
import 'package:pub_dev/service/rate_limit/rate_limit.dart';
import 'package:pub_dev/shared/versions.dart';
Expand Down Expand Up @@ -419,6 +420,20 @@ class PackageBackend {
return count;
}

/// Returns the package options.
Future<api.PkgOptions> getPackageOptions(String package) async {
checkPackageVersionParams(package);
final p = await packageBackend.lookupPackage(package);
if (p == null) {
throw NotFoundException.resource(package);
}
return api.PkgOptions(
isDiscontinued: p.isDiscontinued,
replacedBy: p.replacedBy,
isUnlisted: p.isUnlisted,
);
}

/// Updates [options] on [package].
Future<void> updateOptions(String package, api.PkgOptions options) async {
final authenticatedUser = await requireAuthenticatedWebUser();
Expand Down Expand Up @@ -479,6 +494,7 @@ class PackageBackend {
});
await purgePackageCache(package);
await taskBackend.trackPackage(package);
await apiExporter!.synchronizePackage(package);
}

/// Updates [options] on [package]/[version], assuming the current user
Expand Down Expand Up @@ -517,6 +533,9 @@ class PackageBackend {
}
});
await purgePackageCache(package);
await purgeScorecardData(package, version,
isLatest: pkg.latestVersion == version);
await apiExporter!.synchronizePackage(package);
}

/// Verifies an update to the credential-less publishing settings and
Expand Down Expand Up @@ -781,7 +800,7 @@ class PackageBackend {
if (currentPublisherId != null) {
await purgePublisherCache(publisherId: currentPublisherId);
}

await apiExporter!.synchronizePackage(packageName);
return rs;
}

Expand Down
39 changes: 39 additions & 0 deletions app/lib/scorecard/backend.dart
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,13 @@

import 'dart:async';

import 'package:_pub_shared/data/package_api.dart';
import 'package:clock/clock.dart';
import 'package:gcloud/service_scope.dart' as ss;
import 'package:logging/logging.dart';
import 'package:meta/meta.dart';
import 'package:pool/pool.dart';
import 'package:pub_dev/service/download_counts/backend.dart';
import 'package:pub_dev/service/download_counts/computations.dart';
import 'package:pub_dev/shared/exceptions.dart';
import 'package:pub_dev/task/backend.dart';
Expand Down Expand Up @@ -186,6 +188,43 @@ class ScoreCardBackend {
final pv = list[1] as PackageVersion?;
return PackageStatus.fromModels(p, pv);
}

/// Return the version score object served in the API.
Future<VersionScore> getVersionScore(String package,
{String? version}) async {
final pkg = await packageBackend.lookupPackage(package);
if (pkg == null) {
throw NotFoundException.resource('package "$package"');
}
final v =
(version == null || version == 'latest') ? pkg.latestVersion! : version;
final pv = await packageBackend.lookupPackageVersion(package, v);
if (pv == null) {
throw NotFoundException.resource('package "$package" version "$version"');
}

var updated = pkg.updated;
final card = await scoreCardBackend.getScoreCardData(package, v);
if (updated == null || card.updated?.isAfter(updated) == true) {
updated = card.updated;
}

final tags = <String>{
...pkg.getTags(),
...pv.getTags(),
...?card.derivedTags,
};

return VersionScore(
grantedPoints: card.grantedPubPoints,
maxPoints: card.maxPubPoints,
likeCount: pkg.likes,
downloadCount30Days:
downloadCountsBackend.lookup30DaysTotalCounts(package),
tags: tags.toList(),
lastUpdated: updated,
);
}
}

Future<void> purgeScorecardData(
Expand Down
2 changes: 2 additions & 0 deletions app/lib/task/backend.dart
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import 'package:indexed_blob/indexed_blob.dart' show BlobIndex, FileRange;
import 'package:logging/logging.dart' show Logger;
import 'package:pana/models.dart' show Summary;
import 'package:pool/pool.dart' show Pool;
import 'package:pub_dev/package/api_export/api_exporter.dart';
import 'package:pub_dev/package/backend.dart';
import 'package:pub_dev/package/models.dart';
import 'package:pub_dev/package/upload_signer_service.dart';
Expand Down Expand Up @@ -721,6 +722,7 @@ class TaskBackend {

// Clearing the state cache after the update.
await _purgeCache(package, version);
await apiExporter!.synchronizePackage(package);

// If nothing else is running on the instance, delete it!
// We do this in a microtask after returning, so that it doesn't slow down
Expand Down
12 changes: 10 additions & 2 deletions app/test/admin/exported_api_sync_test.dart
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,8 @@ void main() {
testWithProfile('baseline checks', fn: () async {
await syncExportedApi();
final data = await listExportedApi();
expect(data.keys, hasLength(greaterThan(20)));
expect(data.keys, hasLength(lessThan(40)));
expect(data.keys, hasLength(greaterThan(50)));
expect(data.keys, hasLength(lessThan(70)));

final oxygenFiles = data.keys.where((k) => k.contains('oxygen')).toSet();
expect(oxygenFiles, hasLength(greaterThan(5)));
Expand All @@ -82,12 +82,20 @@ void main() {
'$runtimeVersion/api/archives/oxygen-2.0.0-dev.tar.gz',
'$runtimeVersion/api/packages/oxygen',
'$runtimeVersion/api/packages/oxygen/advisories',
'$runtimeVersion/api/packages/oxygen/likes',
'$runtimeVersion/api/packages/oxygen/options',
'$runtimeVersion/api/packages/oxygen/publisher',
'$runtimeVersion/api/packages/oxygen/score',
'$runtimeVersion/api/packages/oxygen/feed.atom',
'latest/api/archives/oxygen-1.0.0.tar.gz',
'latest/api/archives/oxygen-1.2.0.tar.gz',
'latest/api/archives/oxygen-2.0.0-dev.tar.gz',
'latest/api/packages/oxygen',
'latest/api/packages/oxygen/advisories',
'latest/api/packages/oxygen/likes',
'latest/api/packages/oxygen/options',
'latest/api/packages/oxygen/publisher',
'latest/api/packages/oxygen/score',
'latest/api/packages/oxygen/feed.atom',
});

Expand Down
Loading