Test case for Sidekiq issue #2107

leak.rb

# Minimal example that demonstrates a memory leak in Sidekiq (or Celluloid?).
# Adapted from @grosser example: https://github.com/mperham/sidekiq/blob/master/examples/leak.rb

# Run with LEAK=0 for no memory leak or LEAK=1 for memory leak, like so:
# LEAK=1 sidekiq -r ./leak.rb > /dev/null

require 'sidekiq'

redis = { namespace: :leak }

Sidekiq.configure_client { |config| config.redis = redis }
Sidekiq.configure_server { |config| config.redis = redis }

$counter = 0
$jobs = 5_000

class MyWorker
  include Sidekiq::Worker

  def perform
    report_memory
    check_done

    # Normally, we'd perform some work here:
    # ...

    # Now, assume that the something went wrong for whatever reason and
    # we'd like to retry the job.

    if ENV['LEAK'].to_i == 1
      # If the error is raised using this line, it will cause a memory leak.
      raise Exception.new("Exception - Job failed for some reason")
    else
      # On the other hand, if this line is used, memory usage remains ok.
      raise "RuntimeError - Job failed for some reason"
    end
  end

  def report_memory
    $counter += 1
    if $counter % 100 == 0
      GC.start
      memory = `ps -o rss -p #{Process.pid}`.chomp.split("\n").last.to_i
      $stderr.puts "#{$counter.to_s.ljust(5)} - Memory usage: #{memory}"
    end
  end

  def check_done
    if $counter > $jobs
      exit
    end
  end
end

# Enqueue jobs
$jobs.times { MyWorker.perform_async }

out.txt

---------------------------------------------
Results for LEAK=0 (no or slight memory leak)
---------------------------------------------

$ LEAK=0 sidekiq -r ./leak.rb > /dev/null
102   - Memory usage: 31508
200   - Memory usage: 31696
300   - Memory usage: 31760
400   - Memory usage: 31756
500   - Memory usage: 31752
601   - Memory usage: 31760
701   - Memory usage: 31776
805   - Memory usage: 32008
901   - Memory usage: 31984
1000  - Memory usage: 31928
1100  - Memory usage: 31936
1203  - Memory usage: 32032
1302  - Memory usage: 32040
1403  - Memory usage: 32340
1504  - Memory usage: 32860
1603  - Memory usage: 33856
1701  - Memory usage: 33716
1801  - Memory usage: 33716
1900  - Memory usage: 33780
2000  - Memory usage: 33740
2101  - Memory usage: 33820
2203  - Memory usage: 33716
2300  - Memory usage: 33740
2400  - Memory usage: 33748
2500  - Memory usage: 33732
2601  - Memory usage: 33724
2700  - Memory usage: 33788
2800  - Memory usage: 33716
2902  - Memory usage: 33812
3001  - Memory usage: 33812
3103  - Memory usage: 33892
3204  - Memory usage: 33948
3302  - Memory usage: 33804
3401  - Memory usage: 33968
3501  - Memory usage: 33904
3600  - Memory usage: 33912
3700  - Memory usage: 33904
3802  - Memory usage: 33896
3902  - Memory usage: 33896
4000  - Memory usage: 33896
4101  - Memory usage: 33936
4201  - Memory usage: 33888
4303  - Memory usage: 33896
4401  - Memory usage: 33896
4502  - Memory usage: 33896
4601  - Memory usage: 33896
4700  - Memory usage: 33904
4801  - Memory usage: 33904
4901  - Memory usage: 33928
5000  - Memory usage: 33904

---------------------------------------------
Results for LEAK=1 (big memory leak)
---------------------------------------------

$ LEAK=1 sidekiq -r ./leak.rb > /dev/null

102   - Memory usage: 33016
210   - Memory usage: 37876
301   - Memory usage: 42196
403   - Memory usage: 47672
500   - Memory usage: 50432
603   - Memory usage: 54072
700   - Memory usage: 61288
800   - Memory usage: 66388
900   - Memory usage: 72664
1001  - Memory usage: 75940
1107  - Memory usage: 79448
1202  - Memory usage: 82568
1302  - Memory usage: 86392
1400  - Memory usage: 92520
1500  - Memory usage: 97980
1602  - Memory usage: 105076
1704  - Memory usage: 111540
1803  - Memory usage: 116868
1901  - Memory usage: 120356
2002  - Memory usage: 123564
2100  - Memory usage: 126780
2202  - Memory usage: 130228
2302  - Memory usage: 133076
2402  - Memory usage: 137236
2501  - Memory usage: 140872
2601  - Memory usage: 144988
2700  - Memory usage: 151500
2801  - Memory usage: 158740
2902  - Memory usage: 165884
3001  - Memory usage: 171320
3100  - Memory usage: 176948
3200  - Memory usage: 184184
3300  - Memory usage: 190792
3400  - Memory usage: 196936
3502  - Memory usage: 200432
3600  - Memory usage: 203520
3701  - Memory usage: 206540
3801  - Memory usage: 209800
3900  - Memory usage: 213132
4000  - Memory usage: 216340
4100  - Memory usage: 219336
4203  - Memory usage: 222424
4302  - Memory usage: 226036
4400  - Memory usage: 229084
4500  - Memory usage: 232584
4600  - Memory usage: 236352
4700  - Memory usage: 240272
4801  - Memory usage: 244252
4901  - Memory usage: 248040
5001  - Memory usage: 254120