diff --git a/Gemfile b/Gemfile index 21e152aab..7b19a8a99 100644 --- a/Gemfile +++ b/Gemfile @@ -15,6 +15,7 @@ gem "brotli" gem "bundler" gem "charlock_holmes", ">= 0.7.5" gem "chartkick" +gem "chronic" gem "commonmarker" gem "concurrent-ruby-ext" gem "counter_culture" diff --git a/Gemfile.lock b/Gemfile.lock index ff558cf0d..3ec9120fa 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -97,7 +97,7 @@ GEM ast (2.4.0) autoprefixer-rails (9.5.1.1) execjs - bibliothecary (6.7.4) + bibliothecary (6.8.7) commander deb_control librariesio-gem-parser @@ -130,10 +130,11 @@ GEM activesupport charlock_holmes (0.7.6) chartkick (3.3.1) + chronic (0.10.2) citrus (3.0.2) cliver (0.3.2) coderay (1.1.2) - commander (4.4.7) + commander (4.5.2) highline (~> 2.0.0) commonmarker (0.20.1) ruby-enum (~> 0.5) @@ -200,7 +201,7 @@ GEM rake rake-compiler fast_xs (0.8.0) - ffi (1.11.1) + ffi (1.13.0) fog-aws (3.5.0) fog-core (~> 2.1) fog-json (~> 1.1) @@ -306,7 +307,7 @@ GEM googleapis-common-protos-types (~> 1.0) hashdiff (0.4.0) hashie (3.6.0) - highline (2.0.2) + highline (2.0.3) hiredis (0.6.3) htmlentities (4.3.4) httparty (0.17.0) @@ -406,7 +407,7 @@ GEM org-ruby (0.9.12) rubypants (~> 0.2) os (1.0.1) - ox (2.11.0) + ox (2.13.2) parallel (1.17.0) parser (2.7.0.4) ast (~> 2.4.0) @@ -614,11 +615,11 @@ GEM google-cloud-trace (~> 0.33) stackdriver-core (1.3.3) google-cloud-core (~> 1.2) - strings (0.1.5) + strings (0.1.8) strings-ansi (~> 0.1) unicode-display_width (~> 1.5) unicode_utils (~> 1.4) - strings-ansi (0.1.0) + strings-ansi (0.2.0) terminal-table (1.8.0) unicode-display_width (~> 1.1, >= 1.1.1) thor (0.19.4) @@ -632,14 +633,14 @@ GEM turbolinks (5.2.0) turbolinks-source (~> 5.2) turbolinks-source (5.2.0) - typhoeus (1.3.1) + typhoeus (1.4.0) ethon (>= 0.9.0) tzinfo (1.2.6) thread_safe (~> 0.1) uber (0.1.0) uglifier (4.1.20) execjs (>= 0.3.0, < 3) - unicode-display_width (1.6.0) + unicode-display_width (1.6.1) unicode_utils (1.4.0) vcr (4.0.0) webmock (3.5.1) @@ -674,6 +675,7 @@ DEPENDENCIES bundler charlock_holmes (>= 0.7.5) chartkick + chronic commonmarker concurrent-ruby-ext counter_culture diff --git a/app/models/package_manager/go.rb b/app/models/package_manager/go.rb index 7f6ee9ad2..4539b2487 100644 --- a/app/models/package_manager/go.rb +++ b/app/models/package_manager/go.rb @@ -1,9 +1,11 @@ +# frozen_string_literal: true + module PackageManager class Go < Base - HAS_VERSIONS = false - HAS_DEPENDENCIES = false + HAS_VERSIONS = true + HAS_DEPENDENCIES = true BIBLIOTHECARY_SUPPORT = true - URL = 'http://go-search.org/' + URL = 'https://pkg.go.dev/' COLOR = '#375eab' KNOWN_HOSTS = [ 'bitbucket.org', @@ -21,11 +23,11 @@ class Go < Base def self.package_link(project, version = nil) - "http://go-search.org/view?id=#{project.name}" + "https://pkg.go.dev/#{project.name}#{"@#{version}" if version}" end def self.documentation_url(name, version = nil) - "http://godoc.org/#{name}" + "https://pkg.go.dev/#{name}#{"@#{version}" if version}?tab=doc" end def self.install_instructions(project, version = nil) @@ -33,20 +35,76 @@ def self.install_instructions(project, version = nil) end def self.project_names - get("http://go-search.org/api?action=packages") + # Currently the index only shows the last <=2000 package version releases from the date given. (https://proxy.golang.org/) + project_window = 1.day.ago.strftime("%FT%TZ") + get_raw("https://index.golang.org/index?since=#{project_window}&limit=2000") + .lines + .map { |line| JSON.parse(line)["Path"] } end def self.project(name) - get("http://go-search.org/api?action=package&id=#{name}") + if doc_html = get_html("https://pkg.go.dev/#{name}?tab=doc") + overview_html = get_html("https://pkg.go.dev/#{name}?tab=overview") + + # NB fetching versions from the html only gets dates without timestamps, but we could alternatively use the go proxy too: + # 1) Fetch the list of versions: http://proxy.golang.org/#{module_name}/@v/list + # 2) And for each version, fetch http://proxy.golang.org/#{module_name}/@v/#{v}.info + versions_html = get_html("https://pkg.go.dev/#{name}?tab=versions") + + # Some package pages don't have a Versions tab, but the parent module page may have the Versions tab (e.g. golang.org/x/tools) + if versions_html&.css('.Versions-item').size.zero? && (mod_path = doc_html.css('a[data-test-id="DetailsHeader-infoLabelModule"]').first&.attr('href')) + versions_html = get_html("https://pkg.go.dev/#{mod_path}?tab=versions") + end + + { name: name, html: doc_html, overview_html: overview_html, versions_html: versions_html } + else + { name: name } + end + end + + def self.versions(project, name) + return [] if project.nil? + project[:versions_html]&.css('.Versions-item')&.map do |v| + { number: v.css('a').first.text, published_at: Chronic.parse(v.css('.Versions-commitTime').first.text) } + end end def self.mapping(project) - { - name: project['Package'], - description: project['Synopsis'], - homepage: project['ProjectURL'], - repository_url: get_repository_url(project) - } + if project[:html] + { + name: project[:name], + description: project[:html].css('.Documentation-overview p').map(&:text).join("\n").strip, + licenses: project[:html].css('*[data-test-id="DetailsHeader-infoLabelLicense"] a').map(&:text).join(","), + repository_url: project[:overview_html]&.css('.Overview-sourceCodeLink a')&.first&.text, + homepage: project[:overview_html]&.css('.Overview-sourceCodeLink a')&.first&.text, + versions: project[:versions_html]&.css('.Versions-item')&.map do |v| + { number: v.css('a').first.text, published_at: Chronic.parse(v.css('.Versions-commitTime').first.text) } + end + } + else + { name: project[:name] } + end + end + + def self.dependencies(name, version, _project) + # Go proxy spec: https://golang.org/cmd/go/#hdr-Module_proxy_protocol + # TODO: this can take up to 2sec if it's a cache miss on the proxy. Might be able + # to scrape the webpage or wait for an API for a faster fetch here. + resp = request("https://proxy.golang.org/#{name}/@v/#{version}.mod") + if resp.status == 200 + go_mod_file = resp.body + Bibliothecary::Parsers::Go.parse_go_mod(go_mod_file) + .map do |dep| + { + project_name: dep[:name], + requirements: dep[:requirement], + kind: dep[:type], + platform: "Go", + } + end + else + [] + end end # https://golang.org/cmd/go/#hdr-Import_path_syntax diff --git a/spec/models/package_manager/go_spec.rb b/spec/models/package_manager/go_spec.rb index b82762d2e..571bea10b 100644 --- a/spec/models/package_manager/go_spec.rb +++ b/spec/models/package_manager/go_spec.rb @@ -9,21 +9,17 @@ describe '#package_link' do it 'returns a link to project website' do - expect(described_class.package_link(project)).to eq("http://go-search.org/view?id=foo") + expect(described_class.package_link(project)).to eq("https://pkg.go.dev/foo") end it 'ignores version' do - expect(described_class.package_link(project, '2.0.0')).to eq("http://go-search.org/view?id=foo") + expect(described_class.package_link(project, '2.0.0')).to eq("https://pkg.go.dev/foo@2.0.0") end end describe '#documentation_url' do it 'returns a link to project website' do - expect(described_class.documentation_url('foo')).to eq("http://godoc.org/foo") - end - - it 'ignores version' do - expect(described_class.documentation_url('foo', '2.0.0')).to eq("http://godoc.org/foo") + expect(described_class.documentation_url('foo', '2.0.0')).to eq("https://pkg.go.dev/foo@2.0.0?tab=doc") end end