New download urls (#7576)

* Caching for version listing API * Fix assertions * Fix tests * Fix test * New download URLs for package archives * regenerate code * Fix tests * Regenerate = false
dart-lang · Mar 21, 2024 · cf823e1 · cf823e1
1 parent 14e7e9d
commit cf823e1
Show file tree

Hide file tree

Showing 33 changed files with 173 additions and 128 deletions.
diff --git a/app/lib/frontend/handlers/headers.dart b/app/lib/frontend/handlers/headers.dart
@@ -69,6 +69,12 @@ class CacheHeaders {
     public: true,
   );
 
+  /// Response that redirects to download URL for a package archive
+  static final packageDownloadApi = CacheHeaders._(
+    Duration(hours: 1),
+    public: true,
+  );
+
   /// The package name completion API endpoint serves the cached names
   /// of top packages.
   static final packageNameCompletion = CacheHeaders._(

diff --git a/app/lib/frontend/handlers/pubapi.dart b/app/lib/frontend/handlers/pubapi.dart
@@ -22,6 +22,7 @@ import '../../shared/urls.dart' as urls;
 import '../../task/backend.dart' show taskBackend;
 import 'account.dart';
 import 'custom_api.dart';
+import 'headers.dart';
 import 'listing.dart';
 import 'package.dart';
 import 'report.dart';
@@ -59,14 +60,19 @@ class PubApi {
   /// clients, so while this is deprecated we need to support it indefinitely.
   @EndPoint.get('/api/packages/<package>/versions/<version>/archive.tar.gz')
   @EndPoint.get('/packages/<package>/versions/<version>.tar.gz')
+  @EndPoint.get('/api/archives/<package>-<version>.tar.gz')
   Future<Response> fetchPackage(
     Request request,
     String package,
     String version,
   ) async {
     checkPackageVersionParams(package, version);
     return Response.seeOther(
-        await packageBackend.downloadUrl(package, version));
+      await packageBackend.downloadUrl(package, version),
+      headers: {
+        ...CacheHeaders.packageDownloadApi(),
+      },
+    );
   }
 
   /// Start async upload.

diff --git a/app/lib/frontend/handlers/pubapi.g.dart b/app/lib/frontend/handlers/pubapi.g.dart
diff --git a/app/lib/shared/configuration.dart b/app/lib/shared/configuration.dart
@@ -405,7 +405,7 @@ class Configuration {
       uploadSignerServiceAccount: null,
       blockRobots: true,
       productionHosts: ['localhost'],
-      primaryApiUri: primaryApiUri ?? Uri.parse('https://pub.dartlang.org/'),
+      primaryApiUri: primaryApiUri ?? Uri.parse('https://pub.dev/'),
       primarySiteUri: primarySiteUri ?? Uri.parse('https://pub.dev/'),
       admins: [
         AdminId(

diff --git a/app/lib/shared/urls.dart b/app/lib/shared/urls.dart
@@ -143,8 +143,9 @@ String pkgActivityLogUrl(String package) =>
     pkgPageUrl(package, pkgPageTab: PkgPageTab.activityLog);
 
 String pkgArchiveDownloadUrl(String package, String version, {Uri? baseUri}) {
-  final path =
-      '/packages/${Uri.encodeComponent(package)}/versions/${Uri.encodeComponent(version)}.tar.gz';
+  final p = Uri.encodeComponent(package);
+  final v = Uri.encodeComponent(version);
+  final path = '/api/archives/$p-$v.tar.gz';
   if (baseUri == null) {
     return path;
   } else {

diff --git a/app/lib/tool/test_profile/import_source.dart b/app/lib/tool/test_profile/import_source.dart
@@ -65,8 +65,10 @@ class _PubDevImportSource implements ImportSource {
     final file = File(p.join(archiveCachePath, archiveName));
     // download package archive if not already in the cache
     if (!await file.exists()) {
+      // TODO: Use archive_url from version-listing, that is stable!
       final rs = await _client.get(Uri.parse(
-          '${urls.siteRoot}${urls.pkgArchiveDownloadUrl(package, version)}'));
+        '${urls.siteRoot}/packages/$package/versions/$version.tar.gz',
+      ));
       await file.parent.create(recursive: true);
       await file.writeAsBytes(rs.bodyBytes);
     }

diff --git a/app/test/frontend/golden/pkg_versions_page.html b/app/test/frontend/golden/pkg_versions_page.html
@@ -252,7 +252,7 @@ <h2 id="stable">Stable versions of oxygen</h2>
                           </a>
                         </td>
                         <td class="archive">
-                          <a href="/packages/oxygen/versions/1.2.0.tar.gz" rel="nofollow" title="Download oxygen 1.2.0 archive">
+                          <a href="/api/archives/oxygen-1.2.0.tar.gz" rel="nofollow" title="Download oxygen 1.2.0 archive">
                             <img class="version-table-icon" src="/static/hash-%%etag%%/img/vertical_align_bottom-24px.svg" alt="Download oxygen 1.2.0 archive" width="24" height="24"/>
                           </a>
                         </td>
@@ -270,7 +270,7 @@ <h2 id="stable">Stable versions of oxygen</h2>
                         </td>
                         <td class="documentation"></td>
                         <td class="archive">
-                          <a href="/packages/oxygen/versions/1.0.0.tar.gz" rel="nofollow" title="Download oxygen 1.0.0 archive">
+                          <a href="/api/archives/oxygen-1.0.0.tar.gz" rel="nofollow" title="Download oxygen 1.0.0 archive">
                             <img class="version-table-icon" src="/static/hash-%%etag%%/img/vertical_align_bottom-24px.svg" alt="Download oxygen 1.0.0 archive" width="24" height="24"/>
                           </a>
                         </td>
@@ -315,7 +315,7 @@ <h2 id="prerelease">Prerelease versions of oxygen</h2>
                           </a>
                         </td>
                         <td class="archive">
-                          <a href="/packages/oxygen/versions/2.0.0-dev.tar.gz" rel="nofollow" title="Download oxygen 2.0.0-dev archive">
+                          <a href="/api/archives/oxygen-2.0.0-dev.tar.gz" rel="nofollow" title="Download oxygen 2.0.0-dev archive">
                             <img class="version-table-icon" src="/static/hash-%%etag%%/img/vertical_align_bottom-24px.svg" alt="Download oxygen 2.0.0-dev archive" width="24" height="24"/>
                           </a>
                         </td>

diff --git a/app/test/frontend/handlers/custom_api_test.dart b/app/test/frontend/handlers/custom_api_test.dart
@@ -49,7 +49,7 @@ void main() {
                   'topics': ['chemical-element'],
                 },
                 'archive_url':
-                    '${activeConfiguration.primaryApiUri}/packages/oxygen/versions/1.2.0.tar.gz',
+                    '${activeConfiguration.primaryApiUri}/api/archives/oxygen-1.2.0.tar.gz',
                 'package_url':
                     '${activeConfiguration.primaryApiUri}/api/packages/oxygen',
                 'url':
@@ -80,7 +80,7 @@ void main() {
                   'topics': ['chemical-element'],
                 },
                 'archive_url':
-                    '${activeConfiguration.primaryApiUri}/packages/flutter_titanium/versions/1.10.0.tar.gz',
+                    '${activeConfiguration.primaryApiUri}/api/archives/flutter_titanium-1.10.0.tar.gz',
                 'package_url':
                     '${activeConfiguration.primaryApiUri}/api/packages/flutter_titanium',
                 'url':
@@ -109,7 +109,7 @@ void main() {
                   'topics': ['chemical-element'],
                 },
                 'archive_url':
-                    '${activeConfiguration.primaryApiUri}/packages/neon/versions/1.0.0.tar.gz',
+                    '${activeConfiguration.primaryApiUri}/api/archives/neon-1.0.0.tar.gz',
                 'package_url':
                     '${activeConfiguration.primaryApiUri}/api/packages/neon',
                 'url':

diff --git a/app/test/frontend/handlers/invalid_package_url_test.dart b/app/test/frontend/handlers/invalid_package_url_test.dart
@@ -31,7 +31,7 @@ void main() {
     expect(urls, contains('/packages/<package>/versions/<version>'));
     expect(urls, contains('/api/packages/<package>'));
     // this a naive assertion that fails, if new end-points are introduced!
-    expect(urls, hasLength(43), reason: 'check if new end-points was added');
+    expect(urls, hasLength(44), reason: 'check if new end-points was added');
   });
 
   testWithProfile(

diff --git a/app/test/shared/urls_test.dart b/app/test/shared/urls_test.dart
@@ -193,14 +193,14 @@ void main() {
   group('archive url', () {
     test('without base uri', () {
       expect(pkgArchiveDownloadUrl('foo', '1.0.0+1'),
-          '/packages/foo/versions/1.0.0%2B1.tar.gz');
+          '/api/archives/foo-1.0.0%2B1.tar.gz');
     });
 
     test('with base uri', () {
       expect(
         pkgArchiveDownloadUrl('foo', '1.0.0+1',
             baseUri: Uri.parse('https://pub.dev/')),
-        'https://pub.dev/packages/foo/versions/1.0.0%2B1.tar.gz',
+        'https://pub.dev/api/archives/foo-1.0.0%2B1.tar.gz',
       );
     });
   });

diff --git a/app/test/task/end2end_test.dart b/app/test/task/end2end_test.dart
@@ -235,12 +235,20 @@ void expectGoldenFile(
   expect(xmlContent.split('\n'), golden.split('\n'), reason: 'in $fileName');
 }
 
+final _sdkVersionsToReplace = [
+  runtimeSdkVersion,
+  toolStableDartSdkVersion,
+  Platform.version.split(' ').first,
+];
+
 final _goldenReplacements = <Pattern, String>{
   'Pana <code>$panaVersion</code>,': 'Pana <code>%%pana-version%%</code>,',
-  'Dart <code>$runtimeSdkVersion</code>':
-      'Dart <code>%%stable-dart-version%%</code>',
-  'Dart <code>$toolStableDartSdkVersion</code>':
-      'Dart <code>%%stable-dart-version%%</code>',
+  for (final sdkVersion in _sdkVersionsToReplace)
+    'api.dart.dev/stable/$sdkVersion/':
+        'api.dart.dev/stable/%%stable-dart-version%%/',
+  for (final sdkVersion in _sdkVersionsToReplace)
+    'Dart <code>$sdkVersion</code>':
+        'Dart <code>%%stable-dart-version%%</code>',
   '/static/hash-${staticFileCache.etag}/': '/static/hash-%%etag%%/',
   _timestampPattern: '%%timestamp%%',
   _escapedTimestampPattern: '%%escaped-timestamp%%',

diff --git a/app/test/task/testdata/goldens/documentation/oxygen/1.0.0/oxygen/MainClass-class.html b/app/test/task/testdata/goldens/documentation/oxygen/1.0.0/oxygen/MainClass-class.html
@@ -71,7 +71,7 @@ <h2>Constructors</h2>
                 (
                 <span class="parameter" id="-param-text">
                   <span class="type-annotation">
-                    <a href="https://api.dart.dev/stable/3.3.0/dart-core/String-class.html">String</a>
+                    <a href="https://api.dart.dev/stable/%%stable-dart-version%%/dart-core/String-class.html">String</a>
                   </span>
                   <span class="parameter-name">text</span>
                 </span>
@@ -86,11 +86,11 @@ <h2>Properties</h2>
           <dl class="properties">
             <dt id="hashCode" class="property inherited">
               <span class="name">
-                <a href="https://api.dart.dev/stable/3.3.0/dart-core/Object/hashCode.html">hashCode</a>
+                <a href="https://api.dart.dev/stable/%%stable-dart-version%%/dart-core/Object/hashCode.html">hashCode</a>
               </span>
               <span class="signature">
                 →
-                <a href="https://api.dart.dev/stable/3.3.0/dart-core/int-class.html">int</a>
+                <a href="https://api.dart.dev/stable/%%stable-dart-version%%/dart-core/int-class.html">int</a>
               </span>
             </dt>
             <dd class="inherited">
@@ -102,11 +102,11 @@ <h2>Properties</h2>
             </dd>
             <dt id="runtimeType" class="property inherited">
               <span class="name">
-                <a href="https://api.dart.dev/stable/3.3.0/dart-core/Object/runtimeType.html">runtimeType</a>
+                <a href="https://api.dart.dev/stable/%%stable-dart-version%%/dart-core/Object/runtimeType.html">runtimeType</a>
               </span>
               <span class="signature">
                 →
-                <a href="https://api.dart.dev/stable/3.3.0/dart-core/Type-class.html">Type</a>
+                <a href="https://api.dart.dev/stable/%%stable-dart-version%%/dart-core/Type-class.html">Type</a>
               </span>
             </dt>
             <dd class="inherited">
@@ -122,7 +122,7 @@ <h2>Properties</h2>
               </span>
               <span class="signature">
                 →
-                <a href="https://api.dart.dev/stable/3.3.0/dart-core/String-class.html">String</a>
+                <a href="https://api.dart.dev/stable/%%stable-dart-version%%/dart-core/String-class.html">String</a>
               </span>
             </dt>
             <dd>
@@ -137,14 +137,14 @@ <h2>Methods</h2>
           <dl class="callables">
             <dt id="noSuchMethod" class="callable inherited">
               <span class="name">
-                <a href="https://api.dart.dev/stable/3.3.0/dart-core/Object/noSuchMethod.html">noSuchMethod</a>
+                <a href="https://api.dart.dev/stable/%%stable-dart-version%%/dart-core/Object/noSuchMethod.html">noSuchMethod</a>
               </span>
               <span class="signature">
                 (
                 <wbr/>
                 <span class="parameter" id="noSuchMethod-param-invocation">
                   <span class="type-annotation">
-                    <a href="https://api.dart.dev/stable/3.3.0/dart-core/Invocation-class.html">Invocation</a>
+                    <a href="https://api.dart.dev/stable/%%stable-dart-version%%/dart-core/Invocation-class.html">Invocation</a>
                   </span>
                   <span class="parameter-name">invocation</span>
                 </span>
@@ -168,7 +168,7 @@ <h2>Methods</h2>
                 )
                 <span class="returntype parameter">
                   →
-                  <a href="https://api.dart.dev/stable/3.3.0/dart-core/String-class.html">String</a>
+                  <a href="https://api.dart.dev/stable/%%stable-dart-version%%/dart-core/String-class.html">String</a>
                 </span>
               </span>
             </dt>
@@ -187,7 +187,7 @@ <h2>Methods</h2>
                 )
                 <span class="returntype parameter">
                   →
-                  <a href="https://api.dart.dev/stable/3.3.0/dart-core/String-class.html">String</a>
+                  <a href="https://api.dart.dev/stable/%%stable-dart-version%%/dart-core/String-class.html">String</a>
                 </span>
               </span>
             </dt>
@@ -204,21 +204,21 @@ <h2>Operators</h2>
           <dl class="callables">
             <dt id="operator ==" class="callable inherited">
               <span class="name">
-                <a href="https://api.dart.dev/stable/3.3.0/dart-core/Object/operator_equals.html">operator ==</a>
+                <a href="https://api.dart.dev/stable/%%stable-dart-version%%/dart-core/Object/operator_equals.html">operator ==</a>
               </span>
               <span class="signature">
                 (
                 <wbr/>
                 <span class="parameter" id="==-param-other">
                   <span class="type-annotation">
-                    <a href="https://api.dart.dev/stable/3.3.0/dart-core/Object-class.html">Object</a>
+                    <a href="https://api.dart.dev/stable/%%stable-dart-version%%/dart-core/Object-class.html">Object</a>
                   </span>
                   <span class="parameter-name">other</span>
                 </span>
                 )
                 <span class="returntype parameter">
                   →
-                  <a href="https://api.dart.dev/stable/3.3.0/dart-core/bool-class.html">bool</a>
+                  <a href="https://api.dart.dev/stable/%%stable-dart-version%%/dart-core/bool-class.html">bool</a>
                 </span>
               </span>
             </dt>

diff --git a/app/test/task/testdata/goldens/documentation/oxygen/1.0.0/oxygen/MainClass/MainClass.html b/app/test/task/testdata/goldens/documentation/oxygen/1.0.0/oxygen/MainClass/MainClass.html
@@ -71,7 +71,7 @@ <h1>
             <li>
               <span class="parameter" id="-param-text">
                 <span class="type-annotation">
-                  <a href="https://api.dart.dev/stable/3.3.0/dart-core/String-class.html">String</a>
+                  <a href="https://api.dart.dev/stable/%%stable-dart-version%%/dart-core/String-class.html">String</a>
                 </span>
                 <span class="parameter-name">text</span>
               </span>

diff --git a/app/test/task/testdata/goldens/documentation/oxygen/1.0.0/oxygen/MainClass/text.html b/app/test/task/testdata/goldens/documentation/oxygen/1.0.0/oxygen/MainClass/text.html
@@ -64,7 +64,7 @@ <h1>
           </h1>
         </div>
         <div class="multi-line-signature">
-          <a href="https://api.dart.dev/stable/3.3.0/dart-core/String-class.html">String</a>
+          <a href="https://api.dart.dev/stable/%%stable-dart-version%%/dart-core/String-class.html">String</a>
           <span class="name ">text</span>
           <div class="features">
             <span class="feature">final</span>

diff --git a/app/test/task/testdata/goldens/documentation/oxygen/1.0.0/oxygen/MainClass/toLowerCase.html b/app/test/task/testdata/goldens/documentation/oxygen/1.0.0/oxygen/MainClass/toLowerCase.html
@@ -65,7 +65,7 @@ <h1>
         </div>
         <div class="multi-line-signature">
           <span class="returntype">
-            <a href="https://api.dart.dev/stable/3.3.0/dart-core/String-class.html">String</a>
+            <a href="https://api.dart.dev/stable/%%stable-dart-version%%/dart-core/String-class.html">String</a>
           </span>
           <span class="name ">toLowerCase</span>
           (

diff --git a/app/test/task/testdata/goldens/documentation/oxygen/1.0.0/oxygen/MainClass/toString.html b/app/test/task/testdata/goldens/documentation/oxygen/1.0.0/oxygen/MainClass/toString.html
@@ -68,12 +68,12 @@ <h1>
             <ol class="annotation-list">
               <li>
                 @
-                <a href="https://api.dart.dev/stable/3.3.0/dart-core/override-constant.html">override</a>
+                <a href="https://api.dart.dev/stable/%%stable-dart-version%%/dart-core/override-constant.html">override</a>
               </li>
             </ol>
           </div>
           <span class="returntype">
-            <a href="https://api.dart.dev/stable/3.3.0/dart-core/String-class.html">String</a>
+            <a href="https://api.dart.dev/stable/%%stable-dart-version%%/dart-core/String-class.html">String</a>
           </span>
           <span class="name ">toString</span>
           (
@@ -89,7 +89,7 @@ <h1>
             Some classes have a default textual representation, often paired with a static
             <code>parse</code>
             function (like
-            <a href="https://api.dart.dev/stable/3.3.0/dart-core/int/parse.html" rel="ugc">int.parse</a>
+            <a href="https://api.dart.dev/stable/%%stable-dart-version%%/dart-core/int/parse.html" rel="ugc">int.parse</a>
             ). These classes will provide the textual representation as their string representation.
           </p>
           <p>