diff --git a/app/services/stash_engine/abandoned_dataset_service.rb b/app/services/stash_engine/abandoned_dataset_service.rb index 27bbb4957..fa136210e 100644 --- a/app/services/stash_engine/abandoned_dataset_service.rb +++ b/app/services/stash_engine/abandoned_dataset_service.rb @@ -74,7 +74,7 @@ def send_peer_review_reminders create_activity(reminder_flag, resource) end rescue StandardError => e - p " Exception! #{e.message}" + log " Exception! #{e.message}" end true end @@ -126,7 +126,7 @@ def send_final_withdrawn_notification StashEngine::ResourceMailer.send_final_withdrawn_notification(resource).deliver_now create_activity(reminder_flag, resource) rescue StandardError => e - p " Exception! #{e.message}" + log " Exception! #{e.message}" end true end @@ -145,8 +145,9 @@ def create_activity(flag, resource, status: nil, note: nil) def log(message) return unless logging + return if Rails.env.test? - p message + puts message end def log_data_for_status(status, resource) diff --git a/lib/tasks/dataset_deletion_tasks.rake b/lib/tasks/dataset_deletion_tasks.rake index 04819f622..34b22bd99 100644 --- a/lib/tasks/dataset_deletion_tasks.rake +++ b/lib/tasks/dataset_deletion_tasks.rake @@ -3,32 +3,38 @@ namespace :dataset_deletion do desc 'Send monthly email reminder to the submitter when a dataset has been `in_progress` for more then 1 month' task in_progress_reminders: :environment do - p 'Mailing users whose datasets have been in_progress for more then 1 months' + log 'Mailing users whose datasets have been in_progress for more then 1 months' StashEngine::AbandonedDatasetService.new(logging: true).send_in_progress_reminders end desc 'Send monthly email reminder to the submitter when a dataset has been in `action_required` for more then 1 month' task in_action_required_reminders: :environment do - p 'Mailing users whose datasets have been action_required for more then 1 month' + log 'Mailing users whose datasets have been action_required for more then 1 month' StashEngine::AbandonedDatasetService.new(logging: true).send_action_required_reminders end desc 'Send monthly email reminder to the submitter when a dataset has been in `peer_review` for more then 6 months' task in_peer_review_reminders: :environment do - p 'Mailing users whose datasets have been peer_review for more then 6 months' + log 'Mailing users whose datasets have been peer_review for more then 6 months' StashEngine::AbandonedDatasetService.new(logging: true).send_peer_review_reminders end desc 'Withdraw datasets and send email reminder to the submitter' task auto_withdraw: :environment do - p 'Mailing users whose datasets are being withdrawn' + log 'Mailing users whose datasets are being withdrawn' StashEngine::AbandonedDatasetService.new(logging: true).auto_withdraw end desc 'Send final withdraw email reminder to the submitter' task final_withdrawn_notification: :environment do - p 'Mailing users whose datasets are being withdrawn' + log 'Mailing users whose datasets are being withdrawn' StashEngine::AbandonedDatasetService.new(logging: true).send_final_withdrawn_notification end end + +def log(message) + return if Rails.env.test? + + p message +end # :nocov: diff --git a/lib/tasks/stash_engine_tasks.rake b/lib/tasks/stash_engine_tasks.rake index fb7a38363..c18026140 100644 --- a/lib/tasks/stash_engine_tasks.rake +++ b/lib/tasks/stash_engine_tasks.rake @@ -18,7 +18,7 @@ namespace :identifiers do desc "Update identifiers latest resource if they don't have one" task add_latest_resource: :environment do StashEngine::Identifier.where(latest_resource_id: nil).each do |se_identifier| - puts "Updating identifier #{se_identifier.id}: #{se_identifier}" + log "Updating identifier #{se_identifier.id}: #{se_identifier}" res = StashEngine::Resource.where(identifier_id: se_identifier.id).order(created_at: :desc).first if res.nil? se_identifier.destroy! # useless orphan identifier with no contents which should be deleted @@ -31,7 +31,7 @@ namespace :identifiers do desc 'Add searchable field contents for any identifiers missing it' task add_search: :environment do StashEngine::Identifier.where(search_words: nil).each do |se_identifier| - puts "Updating identifier #{se_identifier} for search" + log "Updating identifier #{se_identifier} for search" se_identifier.update_search_words! end end @@ -42,7 +42,7 @@ namespace :identifiers do license = se_identifier&.latest_resource&.tenant&.default_license next if license.blank? || license == se_identifier.license_id - puts "Updating license to #{license} for #{se_identifier}" + log "Updating license to #{license} for #{se_identifier}" se_identifier.update(license_id: license) end end @@ -84,7 +84,7 @@ namespace :identifiers do desc 'embargo legacy datasets that already had a publication_date in the future -- note that this is somewhat drastic, and may over-embargo items.' task embargo_datasets: :environment do now = Time.now - p "Embargoing resources whose publication_date > '#{now}'" + log "Embargoing resources whose publication_date > '#{now}'" query = <<-SQL SELECT ser.id, ser.identifier_id, seca.user_id FROM stash_engine_identifiers sei @@ -96,7 +96,7 @@ namespace :identifiers do query += " '#{now.strftime('%Y-%m-%d %H:%M:%S')}'" ActiveRecord::Base.connection.execute(query).each do |r| - p "Embargoing: Identifier: #{r[1]}, Resource: #{r[0]}" + log "Embargoing: Identifier: #{r[1]}, Resource: #{r[0]}" StashEngine::CurationActivity.create( resource_id: r[0], user_id: 0, @@ -104,7 +104,7 @@ namespace :identifiers do note: 'Embargo Datasets CRON - publication date has not yet been reached, changing status to `embargo`' ) rescue StandardError => e - p " Exception! #{e.message}" + log " Exception! #{e.message}" next end @@ -113,7 +113,7 @@ namespace :identifiers do desc 'publish datasets based on their publication_date' task publish_datasets: :environment do now = Time.now - p "Publishing resources whose publication_date <= '#{now}'" + log "Publishing resources whose publication_date <= '#{now}'" StashEngine::Resource.need_publishing.find_each do |res| # only release if it's the latest version of the resource @@ -126,7 +126,7 @@ namespace :identifiers do ) rescue StandardError => e # NOTE: we get errors with test data updating DOI and some of the other callbacks on publishing - p " Exception! #{e.message}" + log " Exception! #{e.message}" end end @@ -138,9 +138,9 @@ namespace :identifiers do # This task cleans up datasets that may have had some activity, but they have no real chance of being published. dry_run = args.dry_run == 'true' if dry_run - puts ' ##### remove_abandoned_datasets DRY RUN -- not actually running delete commands' + log ' ##### remove_abandoned_datasets DRY RUN -- not actually running delete commands' else - puts ' ##### remove_abandoned_datasets -- Deleting old versions of datasets that are still in progress' + log ' ##### remove_abandoned_datasets -- Deleting old versions of datasets that are still in progress' end StashEngine::Identifier.where(pub_state: [nil, 'withdrawn', 'unpublished']).find_each do |i| @@ -162,12 +162,12 @@ namespace :identifiers do end if last_user_activity.present? && last_user_activity < 1.year.ago - puts "ABANDONED #{i.identifier} -- #{i.id} -- size #{i.latest_resource.size}" + log "ABANDONED #{i.identifier} -- #{i.id} -- size #{i.latest_resource.size}" if dry_run - puts ' -- skipping deletion due to DRY_RUN setting' + log ' -- skipping deletion due to DRY_RUN setting' else - puts ' -- deleting' + log ' -- deleting' # Perform the actual removal i.resources.each do |r| # Delete temp upload directory, if it exists @@ -205,9 +205,9 @@ namespace :identifiers do # This task cleans up datasets that may have had some activity, but they have no real chance of being published. dry_run = args.dry_run == 'true' if dry_run - puts ' ##### remove_old_versions DRY RUN -- not actually running delete commands' + log ' ##### remove_old_versions DRY RUN -- not actually running delete commands' else - puts ' ##### remove_old_versions -- Deleting old versions of datasets that are still in progress' + log ' ##### remove_old_versions -- Deleting old versions of datasets that are still in progress' end # Remove resources that have been "in progress" for more than a year without updates @@ -217,10 +217,10 @@ namespace :identifiers do ident = res.identifier s3_dir = res.s3_dir_name(type: 'base') - puts "ident #{ident&.id || 'MISSING'} Res #{res.id} -- updated_at #{res.updated_at}" - puts " DESTROY temporary s3 contents #{s3_dir}" + log "ident #{ident&.id || 'MISSING'} Res #{res.id} -- updated_at #{res.updated_at}" + log " DESTROY temporary s3 contents #{s3_dir}" Stash::Aws::S3.new.delete_dir(s3_key: s3_dir) unless dry_run - puts " DESTROY resource #{res.id}" + log " DESTROY resource #{res.id}" res.destroy unless dry_run end @@ -238,19 +238,19 @@ namespace :identifiers do else id_prefix end - puts "checking S3 key #{s3o.key} -- id_prefix #{id_prefix} -- res_id #{res_id}" + log "checking S3 key #{s3o.key} -- id_prefix #{id_prefix} -- res_id #{res_id}" if StashEngine::Resource.exists?(id: res_id) r = StashEngine::Resource.find(res_id) if r.submitted? && (r.zenodo_copies.where("copy_type LIKE 'software%' OR copy_type like 'supp%'").where.not(state: 'finished').count == 0) # if the resource is state == submitted and all zenodo transfers have completed, delete the temporary data - puts " resource is submitted -- DELETE s3 dir #{id_prefix}" + log " resource is submitted -- DELETE s3 dir #{id_prefix}" Stash::Aws::S3.new.delete_dir(s3_key: id_prefix) unless dry_run end else # there is no reasource that corresponds to this S3 dir, so delete the temporary files - puts " resource is deleted -- DELETE s3 dir #{id_prefix}" + log " resource is deleted -- DELETE s3 dir #{id_prefix}" Stash::Aws::S3.new.delete_dir(s3_key: id_prefix) unless dry_run end end @@ -262,11 +262,11 @@ namespace :identifiers do desc 'Set datasets to `submitted` when their peer review period has expired' task expire_peer_review: :environment do now = Date.today - p "Setting resources whose peer_review_end_date <= '#{now}' to 'submitted' curation status" + log "Setting resources whose peer_review_end_date <= '#{now}' to 'submitted' curation status" StashEngine::Resource.where(hold_for_peer_review: true) .where('stash_engine_resources.peer_review_end_date <= ?', now).find_each do |r| if r.current_curation_status == 'peer_review' - p "Expiring peer review for: Identifier: #{r.identifier_id}, Resource: #{r.id}" + log "Expiring peer review for: Identifier: #{r.identifier_id}, Resource: #{r.id}" r.update(hold_for_peer_review: false, peer_review_end_date: nil) StashEngine::CurationActivity.create( resource_id: r.id, @@ -275,18 +275,18 @@ namespace :identifiers do note: 'Expire Peer Review CRON - reached the peer review expiration date, changing status to `submitted`' ) else - p "Removing peer review for: Identifier: #{r.identifier_id}, Resource: #{r.id} due to non-peer_review curation status" + log "Removing peer review for: Identifier: #{r.identifier_id}, Resource: #{r.id} due to non-peer_review curation status" r.update(hold_for_peer_review: false, peer_review_end_date: nil) end rescue StandardError => e - p " Exception! #{e.message}" + log " Exception! #{e.message}" end end desc 'Email the submitter when a dataset has been in `peer_review` past the deadline, and the last reminder was too long ago' task peer_review_reminder: :environment do - p 'Mailing users whose datasets have been in peer_review for a while...' + log 'Mailing users whose datasets have been in peer_review for a while...' StashEngine::Resource.where(hold_for_peer_review: true) .where('stash_engine_resources.peer_review_end_date <= ? OR stash_engine_resources.peer_review_end_date IS NULL', Date.today) .find_each do |r| @@ -296,7 +296,7 @@ namespace :identifiers do if r.current_curation_status == 'peer_review' && r.identifier.latest_resource_id == r.id && (last_reminder.blank? || last_reminder.created_at <= 1.month.ago) - p "Reminding submitter about peer_review dataset. Identifier: #{r.identifier_id}, Resource: #{r.id} updated #{r.updated_at}" + log "Reminding submitter about peer_review dataset. Identifier: #{r.identifier_id}, Resource: #{r.id} updated #{r.updated_at}" StashEngine::UserMailer.peer_review_reminder(r).deliver_now StashEngine::CurationActivity.create( resource_id: r.id, @@ -306,13 +306,13 @@ namespace :identifiers do ) end rescue StandardError => e - p " Exception! #{e.message}" + log " Exception! #{e.message}" end end desc 'Email the submitter 1 time 6 months from publication, when a primary article is not linked' task doi_linking_invitation: :environment do - p 'Mailing users whose datasets have no primary article and were published 6 months ago...' + log 'Mailing users whose datasets have no primary article and were published 6 months ago...' reminder_flag = 'doi_linking_invitation CRON' StashEngine::Identifier.publicly_viewable.find_each do |i| next if i.publication_article_doi @@ -320,7 +320,7 @@ namespace :identifiers do next unless i.date_first_published <= 6.months.ago next if i.latest_resource.nil? - p "Inviting DOI link. Identifier: #{i.id}, Resource: #{i.latest_resource&.id} updated #{i.latest_resource&.updated_at}" + log "Inviting DOI link. Identifier: #{i.id}, Resource: #{i.latest_resource&.id} updated #{i.latest_resource&.updated_at}" StashEngine::UserMailer.doi_invitation(i.latest_resource).deliver_now StashEngine::CurationActivity.create( resource_id: i.latest_resource&.id, @@ -329,14 +329,14 @@ namespace :identifiers do note: "#{reminder_flag} - invited submitter to link an article DOI" ) rescue StandardError => e - p " Exception! #{e.message}" + log " Exception! #{e.message}" end end desc 'Email the submitter when a dataset has been `in_progress` for 3 days' task in_progess_reminder: :environment do - p "Mailing users whose datasets have been in_progress since #{3.days.ago}" + log "Mailing users whose datasets have been in_progress since #{3.days.ago}" StashEngine::Resource.joins(:current_resource_state) .where("stash_engine_resource_states.resource_state = 'in_progress'") .where('stash_engine_resources.updated_at <= ?', 3.days.ago) @@ -344,7 +344,9 @@ namespace :identifiers do reminder_flag = 'in_progress_reminder CRON' if r.curation_activities.where('note LIKE ?', "%#{reminder_flag}%").empty? - p "Mailing submitter about in_progress dataset. Identifier: #{r.identifier_id}, Resource: #{r.id} updated #{r.updated_at}" + if Rails.env.production? + log "Mailing submitter about in_progress dataset. Identifier: #{r.identifier_id}, Resource: #{r.id} updated #{r.updated_at}" + end StashEngine::UserMailer.in_progress_reminder(r).deliver_now StashEngine::CurationActivity.create( resource_id: r.id, @@ -354,7 +356,7 @@ namespace :identifiers do ) end rescue StandardError => e - p " Exception! #{e.message}" + log " Exception! #{e.message}" end end @@ -395,15 +397,15 @@ namespace :identifiers do i.latest_resource.contributors.each do |contrib| next unless contrib.contributor_name == 'National Institutes of Health' - puts "NIH lookup #{contrib.award_number}" + log "NIH lookup #{contrib.award_number}" # - look up the actual grant with the NIH API g = Stash::NIH.find_grant(contrib.award_number) next unless g.present? - puts "NIH found #{g['project_num']}" + log "NIH found #{g['project_num']}" # - see which Institute or Center is the first funder ic = g['agency_ic_fundings'][0]['name'] - puts "NIH funder #{ic}" + log "NIH funder #{ic}" # - replace with the equivalent IC in Dryad Stash::NIH.set_contributor_to_ic(contributor: contrib, ic_name: ic) end @@ -412,7 +414,7 @@ namespace :identifiers do desc 'Curation and publication report' task curation_publication_report: :environment do - p 'Writing curation_publication_report.csv...' + log 'Writing curation_publication_report.csv...' launch_day = Date.new(2019, 9, 17) CSV.open('curation_publication_report.csv', 'w') do |csv| csv << %w[DOI CreatedAt Size NumFiles FileExtensions DaysSubmissionToApproval DaysInCuration] @@ -444,7 +446,7 @@ namespace :identifiers do task datasets_without_primary_articles_report: :environment do FileUtils.mkdir_p(REPORTS_DIR) outfile = File.join(REPORTS_DIR, 'datasets_without_primary_articles.csv') - p "Writing #{outfile}..." + log "Writing #{outfile}..." CSV.open(outfile, 'w') do |csv| csv << %w[DataDOI CreatedAt ISSN Title Authors Institutions Relations] StashEngine::Identifier.publicly_viewable.find_each do |i| @@ -469,7 +471,7 @@ namespace :identifiers do task datasets_with_possible_articles_report: :environment do FileUtils.mkdir_p(REPORTS_DIR) outfile = File.join(REPORTS_DIR, 'datasets_with_possible_articles.csv') - p "Writing #{outfile}..." + log "Writing #{outfile}..." CSV.open(outfile, 'w') do |csv| csv << %w[ID Identifier ISSN] StashEngine::Identifier.publicly_viewable.joins(latest_resource: :resource_publication) @@ -490,7 +492,7 @@ namespace :identifiers do desc 'Generate a report of items associated with common preprint servers' task preprints_report: :environment do - p 'Writing preprints_report.csv...' + log 'Writing preprints_report.csv...' CSV.open('preprints_report.csv', 'w') do |csv| csv << %w[DOI Relation RelatedIdentifierType RelatedIdetifier] visited_identifiers = [] @@ -513,11 +515,11 @@ namespace :identifiers do desc 'Generate a report of the instances when a dataset is in_progress' task in_progress_detail_report: :environment do - puts 'Writting in_progress_detail.csv' + log 'Writting in_progress_detail.csv' CSV.open('in_progress_detail.csv', 'w') do |csv| csv << %w[DOI PubDOI Version DateEnteredIP DateExitedIP StatusExitedTo DatasetSize CurrentStatus EverCurated? EverPublished? Journal WhoPays] StashEngine::Identifier.find_each.with_index do |i, ind| - puts ind if (ind % 100) == 0 + log ind if (ind % 100) == 0 in_ip = false date_entered_ip = i.created_at @@ -570,11 +572,11 @@ namespace :identifiers do desc 'Generate a report of PPR to Curation' task ppr_to_curation_report: :environment do - puts 'Writing ppr_to_curation.csv' + log 'Writing ppr_to_curation.csv' CSV.open('ppr_to_curation.csv', 'w') do |csv| csv << %w[DOI CreatedAt] StashEngine::Identifier.find_each.with_index do |i, ind| - puts ind if (ind % 100) == 0 + log ind if (ind % 100) == 0 ppr_found = false i.resources.map(&:curation_activities).flatten.each do |ca| ppr_found = true if ca.peer_review? @@ -589,11 +591,11 @@ namespace :identifiers do desc 'Generate a detailed report of the instances when a dataset is in PPR' task ppr_detail_report: :environment do - puts 'Writting ppr_detail.csv' + log 'Writting ppr_detail.csv' CSV.open('ppr_detail.csv', 'w') do |csv| csv << %w[DOI PubDOI ManuNumber Version DateEnteredPPR DateExitedPPR StatusExitedTo DatasetSize Journal AutoPPR Integrated WhoPays] StashEngine::Identifier.find_each.with_index do |i, ind| - puts ind if (ind % 100) == 0 + log ind if (ind % 100) == 0 in_ppr = false date_entered_ppr = 'ERROR' @@ -638,7 +640,7 @@ namespace :identifiers do desc 'Generate a report of datasets with associated rejection notices' task rejected_datasets_report: :environment do - puts 'Writing rejected_datasets.csv' + log 'Writing rejected_datasets.csv' CSV.open('rejected_datasets.csv', 'w') do |csv| csv << %w[DOI CreatedAt MSID NumNotifications Published? CurrentStatus] @@ -650,7 +652,7 @@ namespace :identifiers do i = pub.resource.identifier next unless i - puts "MS: #{ms.manuscript_number} identifier #{int_data.first&.identifier_id} same? #{same_manuscripts.size > 1}" + log "MS: #{ms.manuscript_number} identifier #{int_data.first&.identifier_id} same? #{same_manuscripts.size > 1}" csv << [i.identifier, i.created_at, ms.manuscript_number, same_manuscripts.size, i.date_last_published.present?, i.resources.last.current_curation_status] end @@ -664,13 +666,13 @@ namespace :identifiers do # for each void, check if it exists in dryad alert_list = [] voids.each do |invoice_id| - puts "voided invoice #{invoice_id}" + log "voided invoice #{invoice_id}" in_dryad = StashEngine::Identifier.where(payment_id: invoice_id) alert_list << in_dryad.first if in_dryad.present? end if alert_list.present? - puts "Sending alert for identifiers #{alert_list.map(&:id)}" + log "Sending alert for identifiers #{alert_list.map(&:id)}" StashEngine::UserMailer.voided_invoices(alert_list).deliver_now end end @@ -682,13 +684,13 @@ namespace :identifiers do # Get the year-month specified in --year_month argument. # If none, default to the previously completed month. year_month = if args.year_month.blank? - p 'No month specified, assuming last month.' + log 'No month specified, assuming last month.' 1.month.ago.strftime('%Y-%m') else args.year_month end - p "Writing Shopping Cart Report for #{year_month} to file..." + log "Writing Shopping Cart Report for #{year_month} to file..." CSV.open("shopping_cart_report_#{year_month}.csv", 'w') do |csv| csv << %w[DOI CreatedDate CurationStartDate ApprovalDate Size PaymentType PaymentID WaiverBasis InstitutionName @@ -722,12 +724,12 @@ namespace :identifiers do args = Tasks::ArgsParser.parse(:sc_report) # Get the input shopping cart report in --sc_report argument. if args.sc_report.blank? - puts 'Usage: rails deferred_journal_reports -- --sc_report ' + log 'Usage: rails deferred_journal_reports -- --sc_report ' exit end sc_report_file = args.sc_report - puts "Producing deferred journal reports for #{sc_report_file}" + log "Producing deferred journal reports for #{sc_report_file}" sc_report = CSV.parse(File.read(sc_report_file), headers: true) @@ -741,7 +743,7 @@ namespace :identifiers do deferred_filename = "#{md[1]}#{time_period}_deferred_summary.csv" end - puts "Writing summary report to #{deferred_filename}" + log "Writing summary report to #{deferred_filename}" CSV.open(deferred_filename, 'w') do |csv| csv << %w[SponsorName JournalName Count] curr_sponsor = nil @@ -774,16 +776,16 @@ namespace :identifiers do args = Tasks::ArgsParser.parse(:sc_report, :base_report) # Get the input shopping cart report in --base_report and --sc_report arguments. if args.sc_report.blank? || args.base_report.blank? - puts 'Usage: tiered_journal_reports -- --base_report --sc_report ' + log 'Usage: tiered_journal_reports -- --base_report --sc_report ' exit end base_report_file = args.base_report sc_report_file = args.sc_report - puts "Producing tiered journal reports for #{sc_report_file}, using base in #{base_report_file}" + log "Producing tiered journal reports for #{sc_report_file}, using base in #{base_report_file}" base_values = tiered_base_values(base_report_file) - puts "Calculated base values #{base_values}" + log "Calculated base values #{base_values}" sc_report = CSV.parse(File.read(sc_report_file), headers: true) @@ -797,7 +799,7 @@ namespace :identifiers do tiered_filename = "#{md[1]}#{time_period}_tiered_summary.csv" end - puts "Writing summary report to #{tiered_filename}" + log "Writing summary report to #{tiered_filename}" CSV.open(tiered_filename, 'w') do |csv| csv << %w[SponsorName JournalName Count Price] sponsor_summary = [] @@ -886,7 +888,7 @@ namespace :identifiers do return if name.blank? || table.blank? filename = "#{file_prefix}deferred_submissions_#{StashEngine::GenericFile.sanitize_file_name(name)}_#{report_period}.pdf" - puts "Writing sponsor summary to #{filename}" + log "Writing sponsor summary to #{filename}" table_content = '' table.each do |row| table_content << "#{row[0]}#{row[1]}#{row[2]}" @@ -928,7 +930,7 @@ namespace :identifiers do return if name.blank? || table.blank? filename = "#{file_prefix}tiered_submissions_#{StashEngine::GenericFile.sanitize_file_name(name)}_#{report_period}.pdf" - puts "Writing sponsor summary to #{filename}" + log "Writing sponsor summary to #{filename}" table_content = '' table.each do |row| table_content << "#{row[0]}#{row[1]}#{row[2]}" @@ -973,16 +975,16 @@ namespace :identifiers do args = Tasks::ArgsParser.parse(:sc_report, :base_report) # Get the input shopping cart report in --base_report and --sc_report arguments. if args.sc_report.blank? || args.base_report.blank? - puts 'Usage: tiered_tenant_reports -- --base_report --sc_report ' + log 'Usage: tiered_tenant_reports -- --base_report --sc_report ' exit end base_report_file = args.base_report sc_report_file = args.sc_report - puts "Producing tiered tenant reports for #{sc_report_file}, using base in #{base_report_file}" + log "Producing tiered tenant reports for #{sc_report_file}, using base in #{base_report_file}" base_values = tiered_tenant_base_values(base_report_file) - puts "Calculated base values #{base_values}" + log "Calculated base values #{base_values}" sc_report = CSV.parse(File.read(sc_report_file), headers: true) @@ -996,7 +998,7 @@ namespace :identifiers do tiered_filename = "#{md[1]}#{time_period}_tiered_tenant_summary.csv" end - puts "Writing summary report to #{tiered_filename}" + log "Writing summary report to #{tiered_filename}" CSV.open(tiered_filename, 'w') do |csv| csv << %w[SponsorName InstitutionName Count Price] sponsor_summary = [] @@ -1062,13 +1064,13 @@ namespace :identifiers do # Get the year-month specified in --year_month argument. # If none, default to the previously completed month. year_month = if args.year_month.blank? - p 'No month specified, assuming last month.' + log 'No month specified, assuming last month.' 1.month.ago.strftime('%Y-%m') else args.year_month end - p "Writing Geographic Authors Report for #{year_month} to file..." + log "Writing Geographic Authors Report for #{year_month} to file..." CSV.open('geographic_authors_report.csv', 'w') do |csv| csv << ['Dataset DOI', 'Author First', 'Author Last', 'Institution', 'Country'] # Limit the query to datasets that existed at the time of the target report, @@ -1099,7 +1101,7 @@ namespace :identifiers do # If none, default to the previously completed month. if args.year_month.blank? - p 'No month specified, assuming last month.' + log 'No month specified, assuming last month.' year_month = 1.month.ago.strftime('%Y-%m') filename = "dataset_info_report-#{Date.today.strftime('%Y-%m-%d')}.csv" else @@ -1107,7 +1109,7 @@ namespace :identifiers do filename = "dataset_info_report-#{year_month}.csv" end - p "Writing dataset info report to file #{filename}" + log "Writing dataset info report to file #{filename}" CSV.open(filename, 'w') do |csv| csv << ['Dataset DOI', 'Article DOI', 'Approval Date', 'Title', 'Size', 'Institution Name', 'Journal Name'] @@ -1126,16 +1128,16 @@ namespace :identifiers do desc 'populate payment info' task load_payment_info: :environment do - p 'Populating payment information for published/embargoed items' + log 'Populating payment information for published/embargoed items' StashEngine::Identifier.publicly_viewable.where(payment_type: nil).each do |i| i.record_payment - p "#{i.id} #{i.identifier} #{i.payment_type} #{i.payment_id}" + log "#{i.id} #{i.identifier} #{i.payment_type} #{i.payment_id}" end end desc 'populate publicationName' task load_publication_names: :environment do - p "Searching CrossRef and the Journal API for publication names: #{Time.now.utc}" + log "Searching CrossRef and the Journal API for publication names: #{Time.now.utc}" unique_issns = {} StashEngine::Identifier.joins(latest_resource: :resource_publication).where(resource_publication: { publication_name: nil }) .where.not(resource_publication: { publication_issn: nil }).each do |datum| @@ -1147,7 +1149,7 @@ namespace :identifiers do if response.present? && response.parsed_response.present? && response.parsed_response['message'].present? title = response.parsed_response['message']['title'] unique_issns[datum.publication_issn] = title unless unique_issns[datum.publication_issn].present? - p " found title, '#{title}', for #{datum.publication_issn}" + log " found title, '#{title}', for #{datum.publication_issn}" end end datum.update(publicationName: title) unless title.blank? @@ -1156,26 +1158,26 @@ namespace :identifiers do current_resource = identifier.latest_resource_with_public_metadata current_resource.submit_to_solr if current_resource.present? end - p "Finished: #{Time.now.utc}" + log "Finished: #{Time.now.utc}" end desc 'update search words for items that are obviously missing them' task update_missing_search_words: :environment do identifiers = StashEngine::Identifier.where('LENGTH(search_words) < 60 OR search_words IS NULL') - puts "Updating search words for #{identifiers.length} items" + log "Updating search words for #{identifiers.length} items" identifiers.each_with_index do |id, idx| id&.update_search_words! - puts "Updated #{idx + 1}/#{identifiers.length} items" if (idx + 1) % 100 == 0 + log "Updated #{idx + 1}/#{identifiers.length} items" if (idx + 1) % 100 == 0 end end desc 'update search words for all items (in case we need to refresh them all)' task update_all_search_words: :environment do count = StashEngine::Identifier.count - puts "Updating search words for #{count} items" + log "Updating search words for #{count} items" StashEngine::Identifier.find_each.with_index do |id, idx| id&.update_search_words! - puts "Updated #{idx + 1}/#{count} items" if (idx + 1) % 100 == 0 + log "Updated #{idx + 1}/#{count} items" if (idx + 1) % 100 == 0 end end end @@ -1206,7 +1208,7 @@ namespace :curation_stats do CSV.open('curation_timeline_report.csv', 'w') do |csv| csv << %w[DOI CreatedDate CurationStartDate TimesCurated ApprovalDate Size NumFiles FileFormats] StashEngine::Identifier.where(created_at: launch_day..Time.now.utc.to_date).find_each.with_index do |i, idx| - puts("#{idx}/#{datasets.size}") if idx % 100 == 0 + log("#{idx}/#{datasets.size}") if idx % 100 == 0 approval_date_str = i.approval_date&.strftime('%Y-%m-%d') created_date_str = i.created_at&.strftime('%Y-%m-%d') next unless i.resources.submitted.present? @@ -1272,7 +1274,7 @@ namespace :curation_stats do end ids_seen += 1 total_curation_count += curation_count - puts "#{i.id} -- #{curation_count} -- average #{total_curation_count.to_f / ids_seen}" + log "#{i.id} -- #{curation_count} -- average #{total_curation_count.to_f / ids_seen}" end end @@ -1382,7 +1384,7 @@ namespace :journals do journal = StashEngine::Journal.find_by_title(d.publication_name) next unless j.present? - puts "Cleaning journal: #{name}" + log "Cleaning journal: #{name}" StashEngine::Journal.replace_uncontrolled_journal(old_name: d.publication_name, new_journal: journal) end nil @@ -1396,7 +1398,7 @@ namespace :journals do j = StashEngine::Journal.find_by_title(name[0..-2]) next unless j.present? - puts "Cleaning journal: #{name}" + log "Cleaning journal: #{name}" StashEngine::Journal.replace_uncontrolled_journal(old_name: name, new_journal: j) end StashEngine::ResourcePublication.where("publication_name like '%*'").find_each do |d| @@ -1406,7 +1408,7 @@ namespace :journals do j = StashEngine::Journal.find_by_title(name[0..-2]) next unless j.present? - puts "Cleaning journal: #{name}" + log "Cleaning journal: #{name}" StashEngine::Journal.replace_uncontrolled_journal(old_name: name, new_journal: j) end nil @@ -1422,12 +1424,12 @@ namespace :journals do args.dry_run != 'false' end - puts 'Processing with DRY_RUN' if dry_run + log 'Processing with DRY_RUN' if dry_run jj = Stash::Salesforce.db_query("SELECT Id, Name FROM Account where Type='Journal'") jj.find_each do |j| found_journal = StashEngine::Journal.find_by_title(j['Name']) - puts "MISSING from Dryad -- #{j['Name']}" unless found_journal.present? + log "MISSING from Dryad -- #{j['Name']}" unless found_journal.present? end StashEngine::Journal.find_each do |j| @@ -1437,19 +1439,19 @@ namespace :journals do sf_id = Stash::Salesforce.find_account_by_name(j.title) unless sf_id.present? - puts "MISSING from Salesforce -- #{j.title}" + log "MISSING from Salesforce -- #{j.title}" next end sfj = Stash::Salesforce.find(obj_type: 'Account', obj_id: sf_id) if sfj['ISSN__c'] != j.single_issn - puts "Updating ISSN in SF from #{sfj['ISSN__c']} to #{j.single_issn}" + log "Updating ISSN in SF from #{sfj['ISSN__c']} to #{j.single_issn}" Stash::Salesforce.update(obj_type: 'Account', obj_id: sf_id, kv_hash: { ISSN__c: j.single_issn }) unless dry_run end sf_parent_id = sfj['ParentId'] sf_parent = Stash::Salesforce.find(obj_type: 'Account', obj_id: sf_parent_id) - puts "SPONSOR MISMATCH for #{j.single_issn} -- #{j.sponsor&.name} -- #{sf_parent['Name']}" if j.sponsor&.name != sf_parent['Name'] + log "SPONSOR MISMATCH for #{j.single_issn} -- #{j.sponsor&.name} -- #{sf_parent['Name']}" if j.sponsor&.name != sf_parent['Name'] end exit end @@ -1472,5 +1474,10 @@ namespace :journal_email do end end +def log(message) + return if Rails.env.test? + + puts message +end # rubocop:enable Metrics/BlockLength # :nocov: diff --git a/spec/services/stash_engine/abandoned_dataset_service_spec.rb b/spec/services/stash_engine/abandoned_dataset_service_spec.rb index 7729742bb..c00f53e12 100644 --- a/spec/services/stash_engine/abandoned_dataset_service_spec.rb +++ b/spec/services/stash_engine/abandoned_dataset_service_spec.rb @@ -43,7 +43,7 @@ module StashEngine context 'when status date is between 1 month and one year' do [1, 7, 11].each do |month_num| it "sends notification email at #{month_num} months" do - Timecop.travel(month_num.months.from_now) + Timecop.travel(month_num.months.from_now + 1.day) expect(StashEngine::ResourceMailer).to receive_message_chain(:in_progress_delete_notification, :deliver_now).with(resource).with(no_args) expect(subject).to receive(:create_activity).with('in_progress_deletion_notice', resource).once @@ -69,7 +69,7 @@ module StashEngine subject.send(:create_activity, 'in_progress_deletion_notice', resource) expect(StashEngine::ResourceMailer).to receive(:in_progress_delete_notification).once - Timecop.travel(1.month) do + Timecop.travel(1.month + 1.day) do subject.send_in_progress_reminders end end @@ -107,7 +107,7 @@ module StashEngine context 'when status date is between 1 month and one year' do [1, 7, 11].each do |month_num| it "sends notification email at #{month_num} months" do - Timecop.travel(month_num.months.from_now) + Timecop.travel(month_num.months.from_now + 1.day) expect(StashEngine::ResourceMailer).to receive_message_chain(:action_required_delete_notification, :deliver_now).with(resource).with(no_args) @@ -178,7 +178,7 @@ module StashEngine context 'when status date is between 6 months and 1 year' do [6, 7, 9, 11].each do |month_num| it "sends notification email at #{month_num} months" do - Timecop.travel(month_num.months.from_now) + Timecop.travel(month_num.months.from_now + 1.day) expect(StashEngine::ResourceMailer).to receive_message_chain(:peer_review_delete_notification, :deliver_now).with(resource).with(no_args) expect(subject).to receive(:create_activity).with('peer_review_deletion_notice', resource).once @@ -478,22 +478,5 @@ module StashEngine expect(record.user_id).to eq(0) end end - - describe '#log' do - it 'prints message with logging true' do - expect do - subject.class.new(logging: true).send(:log, 'delete_notification_message_with_logs') - end.to output(/delete_notification_message_with_logs/).to_stdout - end - - it 'does not print the message with logging false or missing' do - expect do - subject.class.new(logging: false).send(:log, 'delete_notification_message_with_logs') - end.not_to output(/delete_notification_message_with_logs/).to_stdout - expect do - subject.class.new.send(:log, 'delete_notification_message_with_logs') - end.not_to output(/delete_notification_message_with_logs/).to_stdout - end - end end end diff --git a/spec/shared_examples/resource_email_rake_tasks.rb b/spec/shared_examples/resource_email_rake_tasks.rb index 066db81d7..b2a5d6d69 100644 --- a/spec/shared_examples/resource_email_rake_tasks.rb +++ b/spec/shared_examples/resource_email_rake_tasks.rb @@ -1,7 +1,7 @@ RSpec.shared_examples('send email notifications tasks') do |count, date| it 'should send emails' do - Timecop.travel(date) do + Timecop.travel(date + 1.day) do expect do Rake::Task['identifiers:in_progess_reminder'].execute Rake::Task['dataset_deletion:in_progress_reminders'].execute