Merge pull request #2528 from librariesio/tiegz/go-mod-version-support

Go Modules version and dependency support!

Gemfile

@@ -15,6 +15,7 @@ gem "brotli"
 gem "bundler"
 gem "charlock_holmes", ">= 0.7.5"
 gem "chartkick"
+gem "chronic"
 gem "commonmarker"
 gem "concurrent-ruby-ext"
 gem "counter_culture"

Gemfile.lock

@@ -97,7 +97,7 @@ GEM
     ast (2.4.0)
     autoprefixer-rails (9.5.1.1)
       execjs
-    bibliothecary (6.7.4)
+    bibliothecary (6.8.7)
       commander
       deb_control
       librariesio-gem-parser
@@ -130,10 +130,11 @@ GEM
       activesupport
     charlock_holmes (0.7.6)
     chartkick (3.3.1)
+    chronic (0.10.2)
     citrus (3.0.2)
     cliver (0.3.2)
     coderay (1.1.2)
-    commander (4.4.7)
+    commander (4.5.2)
       highline (~> 2.0.0)
     commonmarker (0.20.1)
       ruby-enum (~> 0.5)
@@ -200,7 +201,7 @@ GEM
       rake
       rake-compiler
     fast_xs (0.8.0)
-    ffi (1.11.1)
+    ffi (1.13.0)
     fog-aws (3.5.0)
       fog-core (~> 2.1)
       fog-json (~> 1.1)
@@ -306,7 +307,7 @@ GEM
       googleapis-common-protos-types (~> 1.0)
     hashdiff (0.4.0)
     hashie (3.6.0)
-    highline (2.0.2)
+    highline (2.0.3)
     hiredis (0.6.3)
     htmlentities (4.3.4)
     httparty (0.17.0)
@@ -406,7 +407,7 @@ GEM
     org-ruby (0.9.12)
       rubypants (~> 0.2)
     os (1.0.1)
-    ox (2.11.0)
+    ox (2.13.2)
     parallel (1.17.0)
     parser (2.7.0.4)
       ast (~> 2.4.0)
@@ -614,11 +615,11 @@ GEM
       google-cloud-trace (~> 0.33)
     stackdriver-core (1.3.3)
       google-cloud-core (~> 1.2)
-    strings (0.1.5)
+    strings (0.1.8)
       strings-ansi (~> 0.1)
       unicode-display_width (~> 1.5)
       unicode_utils (~> 1.4)
-    strings-ansi (0.1.0)
+    strings-ansi (0.2.0)
     terminal-table (1.8.0)
       unicode-display_width (~> 1.1, >= 1.1.1)
     thor (0.19.4)
@@ -632,14 +633,14 @@ GEM
     turbolinks (5.2.0)
       turbolinks-source (~> 5.2)
     turbolinks-source (5.2.0)
-    typhoeus (1.3.1)
+    typhoeus (1.4.0)
       ethon (>= 0.9.0)
     tzinfo (1.2.6)
       thread_safe (~> 0.1)
     uber (0.1.0)
     uglifier (4.1.20)
       execjs (>= 0.3.0, < 3)
-    unicode-display_width (1.6.0)
+    unicode-display_width (1.6.1)
     unicode_utils (1.4.0)
     vcr (4.0.0)
     webmock (3.5.1)
@@ -674,6 +675,7 @@ DEPENDENCIES
   bundler
   charlock_holmes (>= 0.7.5)
   chartkick
+  chronic
   commonmarker
   concurrent-ruby-ext
   counter_culture

app/models/package_manager/go.rb

@@ -1,9 +1,11 @@
+# frozen_string_literal: true
+
 module PackageManager
   class Go < Base
-    HAS_VERSIONS = false
-    HAS_DEPENDENCIES = false
+    HAS_VERSIONS = true
+    HAS_DEPENDENCIES = true
     BIBLIOTHECARY_SUPPORT = true
-    URL = 'http://go-search.org/'
+    URL = 'https://pkg.go.dev/'
     COLOR = '#375eab'
     KNOWN_HOSTS = [
       'bitbucket.org',
@@ -21,32 +23,88 @@ class Go < Base
 
 
     def self.package_link(project, version = nil)
-      "http://go-search.org/view?id=#{project.name}"
+      "https://pkg.go.dev/#{project.name}#{"@#{version}" if version}"
     end
 
     def self.documentation_url(name, version = nil)
-      "http://godoc.org/#{name}"
+      "https://pkg.go.dev/#{name}#{"@#{version}" if version}?tab=doc"
     end
 
     def self.install_instructions(project, version = nil)
       "go get #{project.name}"
     end
 
     def self.project_names
-      get("http://go-search.org/api?action=packages")
+      # Currently the index only shows the last <=2000 package version releases from the date given. (https://proxy.golang.org/)
+      project_window = 1.day.ago.strftime("%FT%TZ")
+      get_raw("https://index.golang.org/index?since=#{project_window}&limit=2000")
+        .lines
+        .map { |line| JSON.parse(line)["Path"] }
     end
 
     def self.project(name)
-      get("http://go-search.org/api?action=package&id=#{name}")
+      if doc_html = get_html("https://pkg.go.dev/#{name}?tab=doc")
+        overview_html = get_html("https://pkg.go.dev/#{name}?tab=overview")
+
+        # NB fetching versions from the html only gets dates without timestamps, but we could alternatively use the go proxy too:
+        #   1) Fetch the list of versions: http://proxy.golang.org/#{module_name}/@v/list
+        #   2) And for each version, fetch http://proxy.golang.org/#{module_name}/@v/#{v}.info
+        versions_html = get_html("https://pkg.go.dev/#{name}?tab=versions")
+
+        # Some package pages don't have a Versions tab, but the parent module page may have the Versions tab (e.g. golang.org/x/tools)
+        if versions_html&.css('.Versions-item').size.zero? && (mod_path = doc_html.css('a[data-test-id="DetailsHeader-infoLabelModule"]').first&.attr('href'))
+          versions_html = get_html("https://pkg.go.dev/#{mod_path}?tab=versions")
+        end
+
+        { name: name, html: doc_html, overview_html: overview_html, versions_html: versions_html }
+      else
+        { name: name }
+      end
+    end
+
+    def self.versions(project, name)
+      return [] if project.nil?
+      project[:versions_html]&.css('.Versions-item')&.map do |v|
+        { number: v.css('a').first.text, published_at: Chronic.parse(v.css('.Versions-commitTime').first.text) }
+      end
     end
 
     def self.mapping(project)
-      {
-        name: project['Package'],
-        description: project['Synopsis'],
-        homepage: project['ProjectURL'],
-        repository_url: get_repository_url(project)
-      }
+      if project[:html]
+        {
+          name: project[:name],
+          description: project[:html].css('.Documentation-overview p').map(&:text).join("\n").strip,
+          licenses: project[:html].css('*[data-test-id="DetailsHeader-infoLabelLicense"] a').map(&:text).join(","),
+          repository_url: project[:overview_html]&.css('.Overview-sourceCodeLink a')&.first&.text,
+          homepage: project[:overview_html]&.css('.Overview-sourceCodeLink a')&.first&.text,
+          versions: project[:versions_html]&.css('.Versions-item')&.map do |v|
+            { number: v.css('a').first.text, published_at: Chronic.parse(v.css('.Versions-commitTime').first.text) }
+          end
+        }
+      else
+        { name: project[:name] }
+      end
+    end
+
+    def self.dependencies(name, version, _project)
+      # Go proxy spec: https://golang.org/cmd/go/#hdr-Module_proxy_protocol
+      # TODO: this can take up to 2sec if it's a cache miss on the proxy. Might be able
+      # to scrape the webpage or wait for an API for a faster fetch here.
+      resp = request("https://proxy.golang.org/#{name}/@v/#{version}.mod")
+      if resp.status == 200
+        go_mod_file = resp.body
+        Bibliothecary::Parsers::Go.parse_go_mod(go_mod_file)
+          .map do |dep|
+            {
+              project_name: dep[:name],
+              requirements: dep[:requirement],
+              kind: dep[:type],
+              platform: "Go",
+            }
+          end
+      else
+        []
+      end
     end
 
     # https://golang.org/cmd/go/#hdr-Import_path_syntax

spec/models/package_manager/go_spec.rb

@@ -9,21 +9,17 @@
 
   describe '#package_link' do
     it 'returns a link to project website' do
-      expect(described_class.package_link(project)).to eq("http://go-search.org/view?id=foo")
+      expect(described_class.package_link(project)).to eq("https://pkg.go.dev/foo")
     end
 
     it 'ignores version' do
-      expect(described_class.package_link(project, '2.0.0')).to eq("http://go-search.org/view?id=foo")
+      expect(described_class.package_link(project, '2.0.0')).to eq("https://pkg.go.dev/foo@2.0.0")
     end
   end
 
   describe '#documentation_url' do
     it 'returns a link to project website' do
-      expect(described_class.documentation_url('foo')).to eq("http://godoc.org/foo")
-    end
-
-    it 'ignores version' do
-      expect(described_class.documentation_url('foo', '2.0.0')).to eq("http://godoc.org/foo")
+      expect(described_class.documentation_url('foo', '2.0.0')).to eq("https://pkg.go.dev/foo@2.0.0?tab=doc")
     end
   end