Skip to content

Commit

Permalink
changed arguments for log progress function
Browse files Browse the repository at this point in the history
  • Loading branch information
davidcam-src committed Aug 26, 2024
1 parent 59145a5 commit df9279d
Showing 1 changed file with 8 additions and 4 deletions.
12 changes: 8 additions & 4 deletions app/services/tasks/download_stats_migration_service.rb
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ def list_work_stat_info(output_path, after_timestamp = nil)
downloads: stat.downloads
}
work_stats_retrieved_from_query_count += 1
log_progress(work_stats_retrieved_from_query_count, total_work_stats)
log_progress(work_stats_retrieved_from_query_count, total_work_stats, 'Retrieval')
end

# Perform aggregation of daily stats ino monthly stats in Ruby, encountered issues with SQL queries
Expand All @@ -38,22 +38,25 @@ def list_work_stat_info(output_path, after_timestamp = nil)
def migrate_to_new_table(csv_path)
csv_data = CSV.read(csv_path, headers: true)
work_stats = csv_data.map { |row| row.to_h.symbolize_keys }
migrated_work_stats_count = 0

Rails.logger.info("Migrating #{work_stats.count} work stats to the new table.")
# Recreate or update objects in new table
work_stats.each do |stat|
create_hyc_download_stat(stat)
migrated_work_stats_count += 1
log_progress(migrated_work_stats_count, work_stats.count, 'Migration')
end
end

private

# Log progress at 25%, 50%, 75%, and 100%
def log_progress(work_stats_count, total_work_stats, is_aggregation = false)
def log_progress(work_stats_count, total_work_stats, process_type = 'Retrieval')
percentages = [0.25, 0.5, 0.75, 1.0]
log_intervals = percentages.map { |percent| (total_work_stats * percent).to_i }
if log_intervals.include?(work_stats_count)
percentage_done = percentages[log_intervals.index(work_stats_count)] * 100
process_type = is_aggregation ? 'Aggregation' : 'Retrieval'
Rails.logger.info("#{process_type} progress: #{percentage_done}% (#{work_stats_count}/#{total_work_stats} work_stats)")
end
end
Expand Down Expand Up @@ -96,7 +99,8 @@ def aggregate_downloads(work_stats)
aggregated_data[key] ||= { file_id: file_id, date: truncated_date, downloads: 0 }
# Sum the downloads for each key
aggregated_data[key][:downloads] += downloads
log_progress(aggregated_work_stats_count, work_stats.count, is_aggregation: true)
aggregated_work_stats_count += 1
log_progress(aggregated_work_stats_count, work_stats.count, 'Aggregation')
end
aggregated_data.values
end
Expand Down

0 comments on commit df9279d

Please sign in to comment.