ruby-on-rails - 在 heroku 上设置 WEB_CONCURRENCY 的公式是什么？

Question

我在 heroku 上托管的 ruby on rails 应用程序经常收到此错误 Memory quota exceeded:

2014-12-29 11:09:37.876 355 <45>1 2014-12-29T11:09:37.210533+00:00 heroku worker.1 - - source=worker.1 dyno=heroku.22144543.00c11b4d-8ec6-46d9-addf-f03163e10f0c sample#memory_total=2899.25MB sample#memory_rss=1023.73MB sample#memory_cache=0.00MB sample#memory_swap=1875.52MB sample#memory_pgpgin=2603236pages sample#memory_pgpgout=2341160pages
2014-12-29 11:09:37.876 132 <4
2014-12-29 11:09:37.876 132 <455>1 2014-12-29T11:09:37.210533+00:00 heroku worker.1 - - Process running mem=2899M(283.1%)>1 2014-12-29T11:09:37.210533+00:00 heroku worker.1 - - Error R14 (Memory quota exceeded)

我阅读了建议 solution 的博客即设置 WEB_CONCURRENCY 配置变量，但我不清楚。

我目前在 heroku 上的配置是:

配置/ unicorn .rb

worker_processes Integer(ENV["WEB_CONCURRENCY"] || 3)
timeout 29
preload_app true

已编辑:

worker 代码

class PhotoWorker
  include Sidekiq::Worker
  sidekiq_options queue: "high"
  # sidekiq_options retry: false

  def perform(params)    
    begin
      puts "Start Woker for #{params.inspect}"
      site = Site.find params['site_id']      
      if params["id"].present?
        photo = Photo.find(params["id"])
        if params['key'].present?
          photo.key = params['key']
          photo.remote_image_url = photo.image.direct_fog_url(:with_path => true)
          photo.image_name = params['key'].split('/').last
          photo.save(validate: false)
          puts photo.inspect
        end     
      else     
        #photo = site.photos.build 
        #photo.hotel_detail_id = site.hotel_detail.id
        #photo.album_id = site.hotel_detail.album.id
        #photo.save(validate: false)
        photo.key = params['key']
        photo.remote_image_url = photo.image.direct_fog_url(:with_path => true)
        #photo.image = params['key']
        #photo.remote_image_url = params['key']
        photo.image_name = params['key'].split('/').last
        puts photo.inspect
        photo.save(validate: false)
      end

      if params["id"].present? && params["crop_x"].present? 
        photo.crop_x = params["crop_x"]
        photo.crop_y = params["crop_y"]
        photo.crop_w = params["crop_w"]
        photo.crop_h = params["crop_h"]
        photo.save(validate: false)
      end


      if params["id"].present? 
        if params['key'].present?
          s3 = AWS::S3.new.buckets[ENV["FOG_DIRECTORY"]]
          s3.objects[params['key']].delete
        end
      else
        AmazonFile.new(params['key']).delete
      end

      puts "Deleted temp file: #{params['key']}" if params['key'].present?
      puts "Photo(#{photo.id}) saved successfully. URL: #{params['key']}"
    rescue Exception => exc
      #puts "Photo not saved. URL: #{params['key']}" 
      puts "Error:  #{exc.message}"
    end      
  end

end

除了 sidekiq，我还在使用 carrierwave_backgrounder gem。

我正在使用 Kraken 进行图像压缩

照片 uploader

class PhotoUploader < CarrierWave::Uploader::Base 
  #include ::CarrierWave::Backgrounder::Delay
  include CarrierWaveDirect::Uploader
  include CarrierWave::MiniMagick
  include CarrierWave::MimeTypes
  process :set_content_type
  process :crop  
  storage :fog
  after :store, :kraken

  version :admin do
    process :resize_to_fit => [200,200]
  end

  #Leisure theme
  version :leisure_358x243, :if => :leisure_theme? do
    process :resize_to_fill => [358,243]
  end
  version :leisure_900x500, :if => :leisure_theme? do
    process :resize_to_fill => [900,500]
  end
 version :leisure_350x147, :if => :leisure_theme? do
    process :resize_to_fill => [350,147]
  end
  version :leisure_1100x344, :if => :leisure_theme? do
    process :resize_to_fill => [1100,344]
  end

  #Business theme
  version :business_360x160, :if => :business_theme? do
    process :resize_to_fill => [360,160]
  end
  version :business_1100x315, :if => :business_theme? do
    process :resize_to_fill => [1100,315]
  end
  version :business_1140x530, :if => :business_theme? do
    process :resize_to_fill => [1140,530]
  end
  version :business_1100x355, :if => :business_theme? do
    process :resize_to_fill => [1100,355]
  end

  #Commthree theme
  version :commthree_550x300, :if => :commthree_theme? do
    process :resize_to_fill => [550,300]
  end
  version :commthree_319x183, :if => :commthree_theme? do
    process :resize_to_fill => [319,183]
  end
  version :commthree_1920x700, :if => :commthree_theme? do
    process :resize_to_fill => [1920,700]
  end

  #All theme
  version :all_360x188 do
    process :resize_to_fill => [360,188]
  end
  version :all_1100x401 do
    process :resize_to_fill => [1100,401]
  end
  version :all_140x88 do    
    process :resize_to_fill => [140,88]
  end

  def kraken(file)
    if version_name.to_s == ""      
      storepath = store_dir + "/" + filename
    else
      fname = filename.split('.')
      #originalfile = "http://s3.amazonaws.com/" + ENV["FOG_DIRECTORY"] + "/" + store_dir  + "/" + fname[0] + "_" + version_name.to_s + "." + fname.last
      storepath = store_dir + "/" + fname[0] + "_" + version_name.to_s + "." + fname.last
    end

    originalfile = "http://s3.amazonaws.com/" + ENV["FOG_DIRECTORY"] + "/" + store_dir + "/" + filename

    kraken = Kraken::API.new(
        :api_key => ENV['KRAKEN_API'],
        :api_secret => ENV['KRAKEN_SECRET']
    )

    params = {}


    Rails.logger.info('Kraken About to Process: ' + originalfile);
    p "version #{version_name}"
    case version_name.to_s
      when "admin"
        params = {
                'lossy' => true,
                'resize' => {
                     'width' => 200,
                     'height' => 200,
                    'strategy' => "auto",
                },
                's3_store' => {
                              'key' => ENV['AWS_ACCESS_KEY_ID'],
                              'secret' => ENV["AWS_SECRET_ACCESS_KEY"],
                              'bucket' => ENV["FOG_DIRECTORY"],
                              'acl' => 'public_read',
                              'path' => storepath
                              },
                  }

      when "leisure_358x243"
        params = {
                'lossy' => true,
                'resize' => {
                    'width' => 358,
                    'height' => 243,
                    'strategy' => "fit",
                },
                's3_store' => {
                              'key' => ENV['AWS_ACCESS_KEY_ID'],
                              'secret' => ENV["AWS_SECRET_ACCESS_KEY"],
                              'bucket' => ENV["FOG_DIRECTORY"],
                              'acl' => 'public_read',
                              'path' => storepath
                              },
                  }
      when "leisure_900x500"
        params = {
                'lossy' => true,
                'resize' => {
                    'width' => 900,
                    'height' => 500,
                    'strategy' => "fit",
                },
                's3_store' => {
                              'key' => ENV['AWS_ACCESS_KEY_ID'],
                              'secret' => ENV["AWS_SECRET_ACCESS_KEY"],
                              'bucket' => ENV["FOG_DIRECTORY"],
                              'acl' => 'public_read',
                              'path' => storepath
                              },
                  }           
      when "leisure_350x147"
        params = {
                'lossy' => true,
                'resize' => {
                    'width' => 350,
                    'height' => 147,
                    'strategy' => "exact",
                },
                's3_store' => {
                              'key' => ENV['AWS_ACCESS_KEY_ID'],
                              'secret' => ENV["AWS_SECRET_ACCESS_KEY"],
                              'bucket' => ENV["FOG_DIRECTORY"],
                              'acl' => 'public_read',
                              'path' => storepath
                              },
                  }
      when "leisure_1100x344"
        params = {
                'lossy' => true,
                'resize' => {
                    'width' => 1100,
                    'height' => 344,
                    'strategy' => "fit",
                },
                's3_store' => {
                              'key' => ENV['AWS_ACCESS_KEY_ID'],
                              'secret' => ENV["AWS_SECRET_ACCESS_KEY"],
                              'bucket' => ENV["FOG_DIRECTORY"],
                              'acl' => 'public_read',
                              'path' => storepath
                              },
                  }

      when "business_360x160"
        params = {
                'lossy' => true,
                'resize' => {
                    'width' => 360,
                    'height' => 160,
                    'strategy' => "fit",
                },
                's3_store' => {
                              'key' => ENV['AWS_ACCESS_KEY_ID'],
                              'secret' => ENV["AWS_SECRET_ACCESS_KEY"],
                              'bucket' => ENV["FOG_DIRECTORY"],
                              'acl' => 'public_read',
                              'path' => storepath
                              },
                  }                
      when "business_1100x315"
        params = {
                'lossy' => true,
                'resize' => {
                    'width' => 1100,
                    'height' => 315,
                    'strategy' => "fit",
                },
                's3_store' => {
                              'key' => ENV['AWS_ACCESS_KEY_ID'],
                              'secret' => ENV["AWS_SECRET_ACCESS_KEY"],
                              'bucket' => ENV["FOG_DIRECTORY"],
                              'acl' => 'public_read',
                              'path' => storepath
                              },
                  }                
      when "business_1140x530"
        params = {
                'lossy' => true,
                'resize' => {
                    'width' => 1140,
                    'height' => 530,
                    'strategy' => "fit",
                },
                's3_store' => {
                              'key' => ENV['AWS_ACCESS_KEY_ID'],
                              'secret' => ENV["AWS_SECRET_ACCESS_KEY"],
                              'bucket' => ENV["FOG_DIRECTORY"],
                              'acl' => 'public_read',
                              'path' => storepath
                              },
                  }
      when "business_1100x355"
        params = {
                'lossy' => true,
                'resize' => {
                    'width' => 1100,
                    'height' => 355,
                    'strategy' => "fit",
                },
                's3_store' => {
                              'key' => ENV['AWS_ACCESS_KEY_ID'],
                              'secret' => ENV["AWS_SECRET_ACCESS_KEY"],
                              'bucket' => ENV["FOG_DIRECTORY"],
                              'acl' => 'public_read',
                              'path' => storepath
                              },
                  }

      when "commthree_550x300"
        params = {
                'lossy' => true,
                'resize' => {
                    'width' => 550,
                    'height' => 300,
                    'strategy' => "fit",
                },
                's3_store' => {
                              'key' => ENV['AWS_ACCESS_KEY_ID'],
                              'secret' => ENV["AWS_SECRET_ACCESS_KEY"],
                              'bucket' => ENV["FOG_DIRECTORY"],
                              'acl' => 'public_read',
                              'path' => storepath
                              },
                  }
      when "commthree_319x183"
        params = {
                'lossy' => true,
                'resize' => {
                    'width' => 319,
                    'height' => 183,
                    'strategy' => "fit",
                },
                's3_store' => {
                              'key' => ENV['AWS_ACCESS_KEY_ID'],
                              'secret' => ENV["AWS_SECRET_ACCESS_KEY"],
                              'bucket' => ENV["FOG_DIRECTORY"],
                              'acl' => 'public_read',
                              'path' => storepath
                              },
                  }
      when "commthree_1920x700"
        params = {
                'lossy' => true,
                'resize' => {
                    'width' => 1920,
                    'height' => 700,
                    'strategy' => "fit",
                },
                's3_store' => {
                              'key' => ENV['AWS_ACCESS_KEY_ID'],
                              'secret' => ENV["AWS_SECRET_ACCESS_KEY"],
                              'bucket' => ENV["FOG_DIRECTORY"],
                              'acl' => 'public_read',
                              'path' => storepath
                              },
                  }

      when "all_360x188"
        params = {
                'lossy' => true,
                'resize' => {
                    'width' => 360,
                    'height' => 188,
                    'strategy' => "fit",
                },
                's3_store' => {
                              'key' => ENV['AWS_ACCESS_KEY_ID'],
                              'secret' => ENV["AWS_SECRET_ACCESS_KEY"],
                              'bucket' => ENV["FOG_DIRECTORY"],
                              'acl' => 'public_read',
                              'path' => storepath
                              },
                  }
      when "all_1100x401"
        params = {
                'lossy' => true,
                'resize' => {
                    'width' => 1100,
                    'height' => 401,
                    'strategy' => "fit",
                },
                's3_store' => {
                              'key' => ENV['AWS_ACCESS_KEY_ID'],
                              'secret' => ENV["AWS_SECRET_ACCESS_KEY"],
                              'bucket' => ENV["FOG_DIRECTORY"],
                              'acl' => 'public_read',
                              'path' => storepath
                              },
                  }
      when "all_140x88"
        params = {
                'lossy' => true,
                'resize' => {
                    'width' => 140,
                    'height' => 88,
                    'strategy' => "fit",
                },
                's3_store' => {
                              'key' => ENV['AWS_ACCESS_KEY_ID'],
                              'secret' => ENV["AWS_SECRET_ACCESS_KEY"],
                              'bucket' => ENV["FOG_DIRECTORY"],
                              'acl' => 'public_read',
                              'path' => storepath
                              },
                  }
    end


    #Store the file online

    if !version_name.blank?

      Rails.logger.info('UPLOADING TO: ' + store_path);

      data = kraken.url(originalfile,params)

      if data.success
          Rails.logger.info('KRAKEN: Success! Optimized image URL: ' + data.kraked_url)
      else
          puts 
          Rails.logger.info('KRAKEN: Fail. Error message: ' + data.message)
      end

    end    
  end

  # Add a white list of extensions which are allowed to be uploaded.
  # For images you might use something like this:
  def extension_white_list
    %w(jpg jpeg gif png)
  end

  # def cache_dir    
  #   "#{Rails.root}/tmp/uploads"
  # end

  def store_dir
    if model.id.present?
      "photos/#{model.id}"
    else
      "photos"
    end
  end

  #file name is missing extension!!!
  def filename
    original_filename if original_filename.present?
  end

  def crop
     puts "i am uploader"
    if model.crop_x.present?
      manipulate! do |img|
        puts "i am cropping"
        cx = model.crop_x.to_i
        cy = model.crop_y.to_i
        cw = model.crop_w.to_i
        ch = model.crop_h.to_i        
        img.crop"#{cw}x#{ch}+#{cx}+#{cy}"
        #img
      end
    end
  end

  def business_theme? (image)
    p "model:#{model.id}"
    (model.hotel_detail.site.theme.layout == "x1")if model.id.present?
  end

  def leisure_theme? (image)
    p "model:#{model.id}"
    (model.hotel_detail.site.theme.layout == "x2")if model.id.present?
  end

  def commthree_theme? (image)
    p "model:#{model.id}"
    (model.hotel_detail.site.theme.layout == "x3") if model.id.present?
  end

  protected
  def secure_token(length=16)
    var = :"@#{mounted_as}_secure_token"
    model.instance_variable_get(var) or model.instance_variable_set(var, SecureRandom.hex(length/2))
  end

end

请建议我如何计算 WEB_CONCURRENCY 的正确值。

Answer 1

看起来您正在使用 minimagick 和 kraken 进行图像处理。如果您完全迁移到 Kraken 并让它们处理您所有的图像处理，您的工作人员的内存应该不会有问题。

删除所有版本，看看是否能提高性能。

version :leisure_358x243, :if => :leisure_theme? do
  process :resize_to_fill => [358,243]
end

如果您想坚持在您的 worker 上处理图像，可以使用比 carrierwave-minimagick 更高效的库。看看carrierwave-vips .