4225 - Migrate Process Document Upload rake to a active job

.github/workflows/ci.yml

@@ -1,5 +1,6 @@
 name: "CI"
 on:
+  push: {}
   pull_request: {}
 jobs:
   test:

Gemfile

@@ -13,16 +13,14 @@ gem 'jquery-rails'
 gem 'jquery-ui-sass-rails'
 gem 'mysql2'
 gem 'nokogiri'
+gem 'postmark-rails'
 gem 'recaptcha', require: 'recaptcha/rails'
 gem 'rmagick'
 gem 'ruby-openai'
 gem 'rvm1-capistrano3', require: false
 gem 'savon', '~> 2.12.0'
 gem 'text'
 gem 'thredded', '~> 1.0'
-gem "recaptcha", require: "recaptcha/rails"
-gem 'ruby-openai'
-gem 'postmark-rails'
 gem 'will_paginate'
 
 gem 'acts_as_list'

app/controllers/application_controller.rb

@@ -50,7 +50,7 @@ def switch_locale(&action)
     end
 
     # append region to locale
-    related_locales = http_accept_language.user_preferred_languages.select do |loc| 
+    related_locales = http_accept_language.user_preferred_languages.select do |loc|
       loc.to_s.include?(locale.to_s) &&                              # is related to the chosen locale (is the locale, or is a regional version of it)
       I18n.available_locales.map{|e| e.to_s}.include?(loc.to_s) # is an available locale
     end
@@ -84,7 +84,7 @@ def guest_user
   def guest_transcription
 
     return head(:forbidden) unless GUEST_TRANSCRIPTION_ENABLED
-    
+
     if check_recaptcha(model: @page, :attribute => :errors)
       User.find(session[:guest_user_id].nil? ? session[:guest_user_id] = create_guest_user.id : session[:guest_user_id])
       redirect_to :controller => 'transcribe', :action => 'display_page', :page_id => @page.id
@@ -198,7 +198,7 @@ def set_friendly_collection(id)
     elsif !DocumentSet.find_by(slug: id).nil?
       @collection = DocumentSet.find_by(slug: id)
     elsif !Collection.find_by(slug: id).nil?
-      @collection = Collection.find_by(slug: id) 
+      @collection = Collection.find_by(slug: id)
     end
 
     # check to make sure URLs haven't gotten scrambled

app/interactors/rake_interactor.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+require 'rake'
+
+class RakeInteractor
+  include Interactor
+
+  def initialize(task_name: '', args: {}, log_file: nil)
+    @task_name = task_name
+    @args      = args
+    @log_file  = log_file || "#{Rails.root}/log/rake.log"
+
+    super
+  end
+
+  def call
+    old_stdout = $stdout
+    log_buffer = StringIO.new
+    $stdout = log_buffer
+
+    begin
+      Rake::Task[@task_name].reenable
+      Rake::Task[@task_name].invoke(*@args.values)
+    rescue => e
+      puts "#{e.class}: #{e.message}"
+      puts e.backtrace.join("\n")
+    ensure
+      $stdout = old_stdout
+
+      File.new(@log_file, 'w').close unless File.exist?(@log_file)
+
+      File.open(@log_file, 'a') do |f|
+        f.puts log_buffer.string
+      end
+    end
+  end
+end

app/interactors/work/refresh_metadata.rb

@@ -19,6 +19,7 @@ def call
     finalize
   rescue => e
     @errors << "Error: #{e}"
+    context.errors = @errors
     context.fail!
   end
 

app/jobs/application_job.rb

@@ -0,0 +1,7 @@
+class ApplicationJob < ActiveJob::Base
+  # Automatically retry jobs that encountered a deadlock
+  # retry_on ActiveRecord::Deadlocked
+
+  # Most jobs are safe to ignore if the underlying records are no longer available
+  # discard_on ActiveJob::DeserializationError
+end

app/jobs/document_upload/process_job.rb

@@ -0,0 +1,8 @@
+class DocumentUpload::ProcessJob < ApplicationJob
+  queue_as :default
+
+  def perform(id, log_file)
+    args = { id: id }
+    RakeInteractor.new(task_name: 'fromthepage:process_document_upload', args: args, log_file: log_file).call
+  end
+end

app/models/document_upload.rb

@@ -21,28 +21,23 @@ class DocumentUpload < ApplicationRecord
   belongs_to :user, optional: true
   belongs_to :collection, optional: true
 
-  validates :collection_id, :file, :presence => true
+  validates :collection_id, :file, presence: true
 
   mount_uploader :file, DocumentUploader
 
-  module Status
-    NEW = 'new'
-    QUEUED = 'queued'
-    PROCESSING = 'processing'
-    FINISHED = 'finished'
-    ERROR = 'error'
-  end
+  enum status: {
+    new: 'new',
+    queued: 'queued',
+    processing: 'processing',
+    finished: 'finished',
+    error: 'error'
+  }, _prefix: :status
 
   def submit_process
-    self.status = Status::QUEUED
+    self.status = :queued
     self.save
-    rake_call = "#{RAKE} fromthepage:process_document_upload[#{self.id}]  --trace >> #{log_file} 2>&1 &"
-
-    # Nice-up the rake call if settings are present
-    rake_call = "nice -n #{NICE_RAKE_LEVEL} " << rake_call if NICE_RAKE_ENABLED
 
-    logger.info rake_call
-    system(rake_call)
+    DocumentUpload::ProcessJob.perform_later(self.id, log_file)
   end
 
   def log_file
@@ -54,6 +49,7 @@ def name
   end
 
   private
+
   def upload_dir
     if self.file && self.file.path
       File.dirname(self.file.path)

app/views/admin/uploads.html.slim

@@ -34,10 +34,10 @@ table.admin-grid.datagrid.striped
               span Actions
               =svg_symbol '#icon-list', class: 'icon'
             dd
-              -if document.status == DocumentUpload::Status::NEW
+              -if document.status == 'new'
                 =link_to(t('.process_upload'), { :action => 'process_upload', :id => document.id })
                 hr
-              -if document.status == DocumentUpload::Status::PROCESSING ||  document.status == DocumentUpload::Status::FINISHED || document.status == DocumentUpload::Status::ERROR
+              -if document.status_processing? || document.status_finished? || document.status_error?
                 =link_to(t('.show_processing_log'), admin_view_processing_log_path(:id => document.id), target: '_blank')
                 hr
               =link_to(t('.delete_upload'), admin_delete_upload_path(:id => document.id), class: 'fgred', data: { confirm: t('.are_you_sure') })

config/application.rb

@@ -46,6 +46,8 @@ class Application < Rails::Application
       end
     end
 
+    # Load rake tasks
+    Rails.application.load_tasks
   end
 
 

lib/tasks/bulk_export.rake

@@ -16,18 +16,18 @@ namespace :fromthepage do
     bulk_export_id = args.bulk_export_id
     print "fetching bulk export with ID=#{bulk_export_id}\n"
     bulk_export = BulkExport.find bulk_export_id
-    
+
     print "found bulk_export for \n\tuser=#{bulk_export.user.login}, \n"
     print "\tfrom collection=#{bulk_export.collection.title}\n" if bulk_export.collection
     pp bulk_export.attributes
-        
+
     bulk_export.status = BulkExport::Status::PROCESSING
     bulk_export.save
-    
-#    process_batch(bulk_export, File.dirname(bulk_export.file.path), bulk_export.id.to_s)
+
+    # process_batch(bulk_export, File.dirname(bulk_export.file.path), bulk_export.id.to_s)
     bulk_export.export_to_zip
 
-    bulk_export.status = DocumentUpload::Status::FINISHED
+    bulk_export.status = :finished
     bulk_export.save