require 'concurrent-edge' # => true # This little bit more complicated commented example aims to # demonstrate some of the capabilities of concurrent-ruby new abstractions. # It is a concurrent processing pipeline which on one side has several web crawlers. # They are searching the web for data and filling buffer. # On the other side there are data processors which are pop the data from buffer. # They are processing the data and storing results into a DB # which has limited concurrency level. # Some of the parts like Web and DB are just stubs. # Each part logs and increments counters to keep some stats about the pipeline. # There is also a periodical readout of the stats into log scheduled. # Schema of the pipeline: # web-crawlers -> buffer -> data-processing -> DB # \____________________________\_____\___> logging # TODO (pitr-ch 10-Mar-2019): replace with a better more realistic example using # * actors for limited concurrency with state - local DB connection # * throttled futures for REST API - limiting server load # The central logger is defined first. # It has state like the logger instance, therefore the actor is used. # It is better to exactly define the communication protocol of the logging actor. # It will only understand these messages. Log = Concurrent::ImmutableStruct.new :severity, :message # => Log SetLevel = Concurrent::ImmutableStruct.new :level # => SetLevel require 'logger' # => false require 'stringio' # => false # Including actor constants so this scope understands ANY etc. include Concurrent::ErlangActor::EnvironmentConstants # => Object # The logger does not need a dedicated thread, let's use a pool. LOGGING = Concurrent::ErlangActor.spawn Logger::FATAL, type: :on_pool, name: 'logger' do |level| # a Logger instance with nicer formatting is created @logger = Logger.new($captured_out) @logger.level = level @logger.formatter = lambda do |severity, datetime, progname, msg| = case msg when String msg when Exception format "%s (%s)\n%s", msg., msg.class, (msg.backtrace || []).join("\n") else msg.inspect end format "[%s] %5s -- %s: %s\n", datetime.strftime('%Y-%m-%d %H:%M:%S.%L'), severity, progname, end # definition of the logging actor behaviour receive( # log messages on(Log) { || @logger.log .severity, . }, # change level on(SetLevel) { || @logger.level = .level }, # It is a good practice to read and log bad messages, # otherwise they would accumulate in the inbox. on(ANY) { || @logger.error bad_message: }, # The logger has static behaviour, therefore keep can be used, and the actor # will behave the same with each message received as defined below. keep: true) end # => #<Concurrent::ErlangActor::Pid:0x000002 logger running> # testing the logger works as expected LOGGING.tell Log[Logger::FATAL, :tornado] # => #<Concurrent::ErlangActor::Pid:0x000002 logger running> LOGGING.tell Log[Logger::INFO, :wind] # => #<Concurrent::ErlangActor::Pid:0x000002 logger running> LOGGING.tell SetLevel[Logger::DEBUG] # => #<Concurrent::ErlangActor::Pid:0x000002 logger running> LOGGING.tell Log[Logger::INFO, :breeze] # => #<Concurrent::ErlangActor::Pid:0x000002 logger running> sleep 0.05 # the logging is asynchronous, we need to wait a bit until it's written get_captured_output # => "[2020-01-26 16:20:23.054] FATAL -- : :tornado\n" + # "[2020-01-26 16:20:23.054] INFO -- : :breeze\n" # the logging could be wrapped in a method def log(severity, ) LOGGING.tell Log[severity, ] true end # => :log include Logger::Severity # => Object log INFO, 'alive' # => true sleep 0.05 # => 0 get_captured_output # => "[2020-01-26 16:20:23.104] INFO -- : alive\n" # The stub which will represent the web module Web @counter = Concurrent::AtomicFixnum.new def self.search sleep 0.01 @counter.increment.to_s(16) end end # The cancellation which will be used to cancel the whole processing pipeline. @cancellation, origin = Concurrent::Cancellation.new # => #<Concurrent::Cancellation:0x000003 pending> # Buffer for work buffer_capacity = 10 # => 10 @buffer = Concurrent::Promises::Channel.new buffer_capacity # => #<Concurrent::Promises::Channel:0x000004 capacity taken 0 of 10> web_crawler_count = 4 # => 4 # Track the number of data provided by each crawler crawler_data_counter = Array.new(web_crawler_count) do |i| # this is accessed by multiple threads so it should be a tread-safe counter Concurrent::AtomicFixnum.new end # the array is frozen which makes it immutable, # therefore safe to use when concurrently accessed. # Otherwise if it was being modified it wound has to be Concurrent::Array to make it safe. crawler_data_counter.freeze # => [#<Concurrent::AtomicFixnum:0x000005 value:0>, # #<Concurrent::AtomicFixnum:0x000006 value:0>, # #<Concurrent::AtomicFixnum:0x000007 value:0>, # #<Concurrent::AtomicFixnum:0x000008 value:0>] # The web crawlers are defined directly with threads to start the example simply. # They search the web and immediately as they find something they push # the data into the buffer. # The push will block if the buffer is full, # regulating how fast is the work being found. # This is called backpressure. crawlers = Array.new web_crawler_count do |i| Thread.new do while true # crawl the web until cancelled break if @cancellation.canceled? # will block and slow down the crawler if the buffer is full data = Web.search until @buffer.push data, 0.1 # It is a good practice to use timeouts on all blocking operations # If the pipeline is cancelled and the data-processors finish # before taking data from buffer a crawler could get stack on this push. break if @cancellation.canceled? end # it pushed data, increment its counter crawler_data_counter[i].increment log DEBUG, "crawler #{i} found #{data}" end end end.freeze # => [#<Thread:0x000009@medium-example.in.rb:130 run>, # #<Thread:0x00000a@medium-example.in.rb:130 run>, # #<Thread:0x00000b@medium-example.in.rb:130 run>, # #<Thread:0x00000c@medium-example.in.rb:130 run>] # So far only the crawlers looking for data are defined # pushing data into the buffer. # The data processing definition follows. # Threads are not used again directly but rather the data processing # is defined using Futures. # Even though that makes the definition more complicated # it has a big advantage that data processors will not require a Thread each # but they will share and run on a Thread pool. # That removes an important limitation of the total number of threads process can have, # which can be an issue in larger systems. # This example would be fine with using the Threads # however it would not demonstrate the more advanced usage then. # The data processing stores results in a DB, # therefore the stub definition of a database precedes the data processing. module DB @data = Concurrent::Map.new # increment a counter for char def self.add(char, count) @data.compute char do |old| (old || 0) + count end true end # return the stored data as Hash def self.data @data.each_pair.reduce({}) { |h, (k, v)| h.update k => v } end end # => :data # Lets assume that instead having this DB # we have limited number of connections # and therefore there is a limit on # how many threads can communicate with the DB at the same time. # The throttle is created to limit the number of concurrent access to DB. @db_throttle = Concurrent::Throttle.new 4 # => #<Concurrent::Throttle:0x00000d capacity available 4 of 4> # The data processing definition follows data_processing_count = 20 # this could actually be thousands if required # track the number of data received by data processors @data_processing_counters = Array.new data_processing_count do Concurrent::AtomicFixnum.new end.freeze # => [#<Concurrent::AtomicFixnum:0x00000e value:0>, # #<Concurrent::AtomicFixnum:0x00000f value:0>, # #<Concurrent::AtomicFixnum:0x000010 value:0>, # #<Concurrent::AtomicFixnum:0x000011 value:0>, # #<Concurrent::AtomicFixnum:0x000012 value:0>, # #<Concurrent::AtomicFixnum:0x000013 value:0>, # #<Concurrent::AtomicFixnum:0x000014 value:0>, # #<Concurrent::AtomicFixnum:0x000015 value:0>, # #<Concurrent::AtomicFixnum:0x000016 value:0>, # #<Concurrent::AtomicFixnum:0x000017 value:0>, # #<Concurrent::AtomicFixnum:0x000018 value:0>, # #<Concurrent::AtomicFixnum:0x000019 value:0>, # #<Concurrent::AtomicFixnum:0x00001a value:0>, # #<Concurrent::AtomicFixnum:0x00001b value:0>, # #<Concurrent::AtomicFixnum:0x00001c value:0>, # #<Concurrent::AtomicFixnum:0x00001d value:0>, # #<Concurrent::AtomicFixnum:0x00001e value:0>, # #<Concurrent::AtomicFixnum:0x00001f value:0>, # #<Concurrent::AtomicFixnum:0x000020 value:0>, # #<Concurrent::AtomicFixnum:0x000021 value:0>] def data_processing(i) # pop_op returns a future which is fulfilled with a message from buffer # when a message is valuable. @buffer.pop_op.then_on(:fast) do |data| # then we process the message on :fast pool since this has no blocking log DEBUG, "data-processor #{i} got #{data}" @data_processing_counters[i].increment sleep 0.1 # simulate it actually doing something which take some time # find the most frequent char data.chars. group_by { |v| v }. map { |ch, arr| [ch, arr.size] }. max_by { |ch, size| size } end.then_on(@db_throttle.on(:io)) do |char, count| # the db access has to be limited therefore the db_throttle is used # DBs use io therefore this part is executed on global thread pool wor :io DB.add char, count end.then_on(:fast) do |_| # last section executes back on :fast executor # checks if it was cancelled # if not then it calls itself recursively # which in combination with #run will turn this into infinite data processing # (until cancelled) # The #run will keep flatting to the inner future as long the value is a future. if @cancellation.canceled? # return something else then future, #run will stop executing :done else # continue running with a future returned by data_processing data_processing i end end end # create the data processors data_processors = Array.new data_processing_count do |i| data_processing(i).run end # => [#<Concurrent::Promises::Future:0x000022 pending>, # #<Concurrent::Promises::Future:0x000023 pending>, # #<Concurrent::Promises::Future:0x000024 pending>, # #<Concurrent::Promises::Future:0x000025 pending>, # #<Concurrent::Promises::Future:0x000026 pending>, # #<Concurrent::Promises::Future:0x000027 pending>, # #<Concurrent::Promises::Future:0x000028 pending>, # #<Concurrent::Promises::Future:0x000029 pending>, # #<Concurrent::Promises::Future:0x00002a pending>, # #<Concurrent::Promises::Future:0x00002b pending>, # #<Concurrent::Promises::Future:0x00002c pending>, # #<Concurrent::Promises::Future:0x00002d pending>, # #<Concurrent::Promises::Future:0x00002e pending>, # #<Concurrent::Promises::Future:0x00002f pending>, # #<Concurrent::Promises::Future:0x000030 pending>, # #<Concurrent::Promises::Future:0x000031 pending>, # #<Concurrent::Promises::Future:0x000032 pending>, # #<Concurrent::Promises::Future:0x000033 pending>, # #<Concurrent::Promises::Future:0x000034 pending>, # #<Concurrent::Promises::Future:0x000035 pending>] # Some statics are collected in crawler_data_counter # and @data_processing_counters. # Schedule a periodical readout to a log. def readout(crawler_data_counter) # schedule readout in 0.4 sec or on cancellation (@cancellation.origin | Concurrent::Promises.schedule(0.4)).then do log INFO, "\ncrawlers found: #{crawler_data_counter.map(&:value).join(', ')}\n" + "data processors consumed: #{@data_processing_counters.map(&:value).join(', ')}" end.then do # reschedule if not cancelled readout crawler_data_counter unless @cancellation.canceled? end end # => :readout # start the periodical readouts readouts = readout(crawler_data_counter).run # => #<Concurrent::Promises::Future:0x000036 pending> sleep 2 # let the whole processing pipeline work # cancel everything origin.resolve # => #<Concurrent::Promises::ResolvableEvent:0x000037 resolved> # wait for everything to stop crawlers.each(&:join) # => [#<Thread:0x000009@medium-example.in.rb:130 dead>, # #<Thread:0x00000a@medium-example.in.rb:130 dead>, # #<Thread:0x00000b@medium-example.in.rb:130 dead>, # #<Thread:0x00000c@medium-example.in.rb:130 dead>] data_processors.each(&:wait!)[0..10] # => [#<Concurrent::Promises::Future:0x000022 fulfilled with :done>, # #<Concurrent::Promises::Future:0x000023 fulfilled with :done>, # #<Concurrent::Promises::Future:0x000024 fulfilled with :done>, # #<Concurrent::Promises::Future:0x000025 fulfilled with :done>, # #<Concurrent::Promises::Future:0x000026 fulfilled with :done>, # #<Concurrent::Promises::Future:0x000027 fulfilled with :done>, # #<Concurrent::Promises::Future:0x000028 fulfilled with :done>, # #<Concurrent::Promises::Future:0x000029 fulfilled with :done>, # #<Concurrent::Promises::Future:0x00002a fulfilled with :done>, # #<Concurrent::Promises::Future:0x00002b fulfilled with :done>, # #<Concurrent::Promises::Future:0x00002c fulfilled with :done>] readouts.wait! # => #<Concurrent::Promises::Future:0x000036 fulfilled with nil> # terminate the logger Concurrent::ErlangActor.terminate LOGGING, :cancelled # => true LOGGING.terminated.wait # => #<Concurrent::Promises::Future:0x000038 rejected with :cancelled> # inspect collected char frequencies DB.data # => {"3"=>18, # "4"=>18, # "1"=>18, # "2"=>18, # "5"=>18, # "6"=>18, # "8"=>18, # "7"=>18, # "9"=>18, # "a"=>18, # "b"=>18, # "c"=>18, # "f"=>3, # "d"=>18, # "e"=>18} # see the logger output get_captured_output # => "[2020-01-26 16:20:23.174] DEBUG -- : crawler 1 found 1\n" + # "[2020-01-26 16:20:23.174] DEBUG -- : crawler 2 found 2\n" + # "[2020-01-26 16:20:23.175] DEBUG -- : crawler 3 found 3\n" + # "[2020-01-26 16:20:23.175] DEBUG -- : crawler 0 found 4\n" + # "[2020-01-26 16:20:23.176] DEBUG -- : data-processor 0 got 1\n" + # "[2020-01-26 16:20:23.176] DEBUG -- : data-processor 1 got 2\n" + # "[2020-01-26 16:20:23.176] DEBUG -- : data-processor 2 got 3\n" + # "[2020-01-26 16:20:23.176] DEBUG -- : data-processor 3 got 4\n" + # "[2020-01-26 16:20:23.186] DEBUG -- : crawler 2 found 5\n" + # "[2020-01-26 16:20:23.186] DEBUG -- : crawler 1 found 6\n" + # "[2020-01-26 16:20:23.187] DEBUG -- : crawler 3 found 7\n" + # "[2020-01-26 16:20:23.187] DEBUG -- : data-processor 4 got 5\n" + # "[2020-01-26 16:20:23.187] DEBUG -- : crawler 0 found 8\n" + # "[2020-01-26 16:20:23.188] DEBUG -- : data-processor 5 got 6\n" + # "[2020-01-26 16:20:23.188] DEBUG -- : data-processor 6 got 7\n" + # "[2020-01-26 16:20:23.188] DEBUG -- : data-processor 7 got 8\n" + # "[2020-01-26 16:20:23.196] DEBUG -- : crawler 2 found 9\n" + # "[2020-01-26 16:20:23.196] DEBUG -- : crawler 1 found a\n" + # "[2020-01-26 16:20:23.196] DEBUG -- : data-processor 8 got 9\n" + # "[2020-01-26 16:20:23.196] DEBUG -- : crawler 3 found b\n" + # "[2020-01-26 16:20:23.197] DEBUG -- : data-processor 9 got a\n" + # "[2020-01-26 16:20:23.197] DEBUG -- : data-processor 10 got b\n" + # "[2020-01-26 16:20:23.198] DEBUG -- : crawler 0 found c\n" + # "[2020-01-26 16:20:23.198] DEBUG -- : data-processor 11 got c\n" + # "[2020-01-26 16:20:23.206] DEBUG -- : crawler 2 found d\n" + # "[2020-01-26 16:20:23.206] DEBUG -- : crawler 3 found e\n" + # "[2020-01-26 16:20:23.207] DEBUG -- : crawler 1 found f\n" + # "[2020-01-26 16:20:23.209] DEBUG -- : crawler 0 found 10\n" + # "[2020-01-26 16:20:23.217] DEBUG -- : crawler 3 found 11\n" + # "[2020-01-26 16:20:23.218] DEBUG -- : crawler 2 found 12\n" + # "[2020-01-26 16:20:23.218] DEBUG -- : crawler 1 found 13\n" + # "[2020-01-26 16:20:23.219] DEBUG -- : crawler 0 found 14\n" + # "[2020-01-26 16:20:23.227] DEBUG -- : crawler 3 found 15\n" + # "[2020-01-26 16:20:23.229] DEBUG -- : crawler 2 found 16\n" + # "[2020-01-26 16:20:23.229] DEBUG -- : crawler 1 found 17\n" + # "[2020-01-26 16:20:23.230] DEBUG -- : crawler 0 found 18\n" + # "[2020-01-26 16:20:23.237] DEBUG -- : crawler 3 found 19\n" + # "[2020-01-26 16:20:23.239] DEBUG -- : crawler 2 found 1a\n" + # "[2020-01-26 16:20:23.239] DEBUG -- : crawler 1 found 1b\n" + # "[2020-01-26 16:20:23.240] DEBUG -- : crawler 0 found 1c\n" + # "[2020-01-26 16:20:23.249] DEBUG -- : crawler 3 found 1d\n" + # "[2020-01-26 16:20:23.250] DEBUG -- : crawler 2 found 1e\n" + # "[2020-01-26 16:20:23.276] DEBUG -- : data-processor 12 got d\n" + # "[2020-01-26 16:20:23.277] DEBUG -- : data-processor 13 got e\n" + # "[2020-01-26 16:20:23.277] DEBUG -- : data-processor 14 got f\n" + # "[2020-01-26 16:20:23.278] DEBUG -- : data-processor 15 got 10\n" + # "[2020-01-26 16:20:23.292] DEBUG -- : data-processor 16 got 11\n" + # "[2020-01-26 16:20:23.293] DEBUG -- : data-processor 17 got 12\n" + # "[2020-01-26 16:20:23.293] DEBUG -- : data-processor 18 got 13\n" + # "[2020-01-26 16:20:23.293] DEBUG -- : data-processor 19 got 14\n" + # "[2020-01-26 16:20:23.301] DEBUG -- : data-processor 2 got 15\n" + # "[2020-01-26 16:20:23.302] DEBUG -- : data-processor 3 got 16\n" + # "[2020-01-26 16:20:23.302] DEBUG -- : crawler 1 found 1f\n" + # "[2020-01-26 16:20:23.303] DEBUG -- : crawler 0 found 20\n" + # "[2020-01-26 16:20:23.303] DEBUG -- : crawler 3 found 21\n" + # "[2020-01-26 16:20:23.303] DEBUG -- : crawler 2 found 22\n" + # "[2020-01-26 16:20:23.304] DEBUG -- : data-processor 0 got 17\n" + # "[2020-01-26 16:20:23.304] DEBUG -- : data-processor 1 got 18\n" + # "[2020-01-26 16:20:23.311] DEBUG -- : crawler 2 found 23\n" + # "[2020-01-26 16:20:23.312] DEBUG -- : crawler 1 found 24\n" + # "[2020-01-26 16:20:23.312] DEBUG -- : crawler 3 found 25\n" + # "[2020-01-26 16:20:23.312] DEBUG -- : crawler 0 found 26\n" + # "[2020-01-26 16:20:23.378] DEBUG -- : data-processor 4 got 19\n" + # "[2020-01-26 16:20:23.380] DEBUG -- : data-processor 5 got 1a\n" + # "[2020-01-26 16:20:23.380] DEBUG -- : data-processor 7 got 1b\n" + # "[2020-01-26 16:20:23.380] DEBUG -- : data-processor 6 got 1c\n" + # "[2020-01-26 16:20:23.393] DEBUG -- : data-processor 8 got 1d\n" + # "[2020-01-26 16:20:23.393] DEBUG -- : data-processor 9 got 1e\n" + # "[2020-01-26 16:20:23.394] DEBUG -- : data-processor 10 got 1f\n" + # "[2020-01-26 16:20:23.394] DEBUG -- : crawler 1 found 27\n" + # "[2020-01-26 16:20:23.394] DEBUG -- : crawler 3 found 28\n" + # "[2020-01-26 16:20:23.394] DEBUG -- : crawler 0 found 29\n" + # "[2020-01-26 16:20:23.395] DEBUG -- : crawler 2 found 2a\n" + # "[2020-01-26 16:20:23.395] DEBUG -- : data-processor 11 got 20\n" + # "[2020-01-26 16:20:23.403] DEBUG -- : crawler 1 found 2b\n" + # "[2020-01-26 16:20:23.404] DEBUG -- : crawler 3 found 2c\n" + # "[2020-01-26 16:20:23.404] DEBUG -- : crawler 0 found 2d\n" + # "[2020-01-26 16:20:23.404] DEBUG -- : crawler 2 found 2e\n" + # "[2020-01-26 16:20:23.404] DEBUG -- : data-processor 14 got 21\n" + # "[2020-01-26 16:20:23.405] DEBUG -- : data-processor 12 got 22\n" + # "[2020-01-26 16:20:23.405] DEBUG -- : data-processor 13 got 23\n" + # "[2020-01-26 16:20:23.405] DEBUG -- : data-processor 15 got 24\n" + # "[2020-01-26 16:20:23.478] DEBUG -- : data-processor 16 got 25\n" + # "[2020-01-26 16:20:23.479] DEBUG -- : crawler 3 found 2f\n" + # "[2020-01-26 16:20:23.479] DEBUG -- : crawler 0 found 30\n" + # "[2020-01-26 16:20:23.479] DEBUG -- : crawler 1 found 31\n" + # "[2020-01-26 16:20:23.479] DEBUG -- : crawler 2 found 32\n" + # "[2020-01-26 16:20:23.480] DEBUG -- : data-processor 17 got 26\n" + # "[2020-01-26 16:20:23.480] DEBUG -- : data-processor 18 got 27\n" + # "[2020-01-26 16:20:23.481] DEBUG -- : data-processor 19 got 28\n" + # "[2020-01-26 16:20:23.489] DEBUG -- : crawler 3 found 33\n" + # "[2020-01-26 16:20:23.489] DEBUG -- : crawler 2 found 34\n" + # "[2020-01-26 16:20:23.490] DEBUG -- : crawler 1 found 35\n" + # "[2020-01-26 16:20:23.490] DEBUG -- : crawler 0 found 36\n" + # "[2020-01-26 16:20:23.498] DEBUG -- : data-processor 1 got 29\n" + # "[2020-01-26 16:20:23.498] DEBUG -- : data-processor 2 got 2a\n" + # "[2020-01-26 16:20:23.499] DEBUG -- : data-processor 3 got 2b\n" + # "[2020-01-26 16:20:23.499] DEBUG -- : data-processor 0 got 2c\n" + # "[2020-01-26 16:20:23.505] DEBUG -- : data-processor 4 got 2d\n" + # "[2020-01-26 16:20:23.506] DEBUG -- : crawler 0 found 37\n" + # "[2020-01-26 16:20:23.507] DEBUG -- : crawler 2 found 38\n" + # "[2020-01-26 16:20:23.507] DEBUG -- : crawler 1 found 39\n" + # "[2020-01-26 16:20:23.507] DEBUG -- : crawler 3 found 3a\n" + # "[2020-01-26 16:20:23.508] DEBUG -- : data-processor 5 got 2e\n" + # "[2020-01-26 16:20:23.508] DEBUG -- : data-processor 7 got 2f\n" + # "[2020-01-26 16:20:23.508] DEBUG -- : data-processor 6 got 30\n" + # "[2020-01-26 16:20:23.515] DEBUG -- : crawler 2 found 3b\n" + # "[2020-01-26 16:20:23.516] DEBUG -- : crawler 1 found 3c\n" + # "[2020-01-26 16:20:23.516] DEBUG -- : crawler 0 found 3d\n" + # "[2020-01-26 16:20:23.517] DEBUG -- : crawler 3 found 3e\n" + # "[2020-01-26 16:20:23.565] INFO -- : \n" + # "crawlers found: 15, 15, 16, 16\n" + # "data processors consumed: 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2\n" + # "[2020-01-26 16:20:23.579] DEBUG -- : data-processor 9 got 31\n" + # "[2020-01-26 16:20:23.580] DEBUG -- : data-processor 8 got 32\n" + # "[2020-01-26 16:20:23.581] DEBUG -- : data-processor 10 got 33\n" + # "[2020-01-26 16:20:23.581] DEBUG -- : data-processor 11 got 34\n" + # "[2020-01-26 16:20:23.601] DEBUG -- : data-processor 13 got 35\n" + # "[2020-01-26 16:20:23.602] DEBUG -- : crawler 0 found 3f\n" + # "[2020-01-26 16:20:23.602] DEBUG -- : crawler 1 found 40\n" + # "[2020-01-26 16:20:23.602] DEBUG -- : crawler 2 found 41\n" + # "[2020-01-26 16:20:23.603] DEBUG -- : data-processor 12 got 36\n" + # "[2020-01-26 16:20:23.603] DEBUG -- : data-processor 14 got 37\n" + # "[2020-01-26 16:20:23.603] DEBUG -- : data-processor 15 got 38\n" + # "[2020-01-26 16:20:23.604] DEBUG -- : crawler 3 found 42\n" + # "[2020-01-26 16:20:23.605] DEBUG -- : data-processor 16 got 39\n" + # "[2020-01-26 16:20:23.606] DEBUG -- : data-processor 17 got 3a\n" + # "[2020-01-26 16:20:23.607] DEBUG -- : data-processor 18 got 3b\n" + # "[2020-01-26 16:20:23.608] DEBUG -- : data-processor 19 got 3c\n" + # "[2020-01-26 16:20:23.611] DEBUG -- : crawler 2 found 43\n" + # "[2020-01-26 16:20:23.612] DEBUG -- : crawler 1 found 44\n" + # "[2020-01-26 16:20:23.613] DEBUG -- : crawler 0 found 45\n" + # "[2020-01-26 16:20:23.613] DEBUG -- : crawler 3 found 46\n" + # "[2020-01-26 16:20:23.682] DEBUG -- : data-processor 0 got 3d\n" + # "[2020-01-26 16:20:23.682] DEBUG -- : data-processor 1 got 3e\n" + # "[2020-01-26 16:20:23.682] DEBUG -- : crawler 1 found 47\n" + # "[2020-01-26 16:20:23.683] DEBUG -- : crawler 0 found 48\n" + # "[2020-01-26 16:20:23.683] DEBUG -- : crawler 3 found 49\n" + # "[2020-01-26 16:20:23.684] DEBUG -- : crawler 2 found 4a\n" + # "[2020-01-26 16:20:23.684] DEBUG -- : data-processor 2 got 3f\n" + # "[2020-01-26 16:20:23.685] DEBUG -- : data-processor 3 got 40\n" + # "[2020-01-26 16:20:23.696] DEBUG -- : crawler 1 found 4b\n" + # "[2020-01-26 16:20:23.696] DEBUG -- : crawler 0 found 4c\n" + # "[2020-01-26 16:20:23.696] DEBUG -- : crawler 3 found 4d\n" + # "[2020-01-26 16:20:23.697] DEBUG -- : crawler 2 found 4e\n" + # "[2020-01-26 16:20:23.701] DEBUG -- : data-processor 4 got 41\n" + # "[2020-01-26 16:20:23.702] DEBUG -- : data-processor 5 got 42\n" + # "[2020-01-26 16:20:23.702] DEBUG -- : data-processor 6 got 43\n" + # "[2020-01-26 16:20:23.702] DEBUG -- : data-processor 7 got 44\n" + # "[2020-01-26 16:20:23.706] DEBUG -- : data-processor 9 got 45\n" + # "[2020-01-26 16:20:23.706] DEBUG -- : crawler 3 found 4f\n" + # "[2020-01-26 16:20:23.707] DEBUG -- : crawler 1 found 50\n" + # "[2020-01-26 16:20:23.707] DEBUG -- : crawler 0 found 51\n" + # "[2020-01-26 16:20:23.707] DEBUG -- : crawler 2 found 52\n" + # "[2020-01-26 16:20:23.709] DEBUG -- : data-processor 8 got 46\n" + # "[2020-01-26 16:20:23.709] DEBUG -- : data-processor 10 got 47\n" + # "[2020-01-26 16:20:23.710] DEBUG -- : data-processor 11 got 48\n" + # "[2020-01-26 16:20:23.717] DEBUG -- : crawler 3 found 53\n" + # "[2020-01-26 16:20:23.718] DEBUG -- : crawler 1 found 54\n" + # "[2020-01-26 16:20:23.718] DEBUG -- : crawler 2 found 55\n" + # "[2020-01-26 16:20:23.719] DEBUG -- : crawler 0 found 56\n" + # "[2020-01-26 16:20:23.782] DEBUG -- : data-processor 12 got 49\n" + # "[2020-01-26 16:20:23.782] DEBUG -- : data-processor 15 got 4a\n" + # "[2020-01-26 16:20:23.783] DEBUG -- : data-processor 14 got 4b\n" + # "[2020-01-26 16:20:23.783] DEBUG -- : data-processor 13 got 4c\n" + # "[2020-01-26 16:20:23.804] DEBUG -- : data-processor 16 got 4d\n" + # "[2020-01-26 16:20:23.804] DEBUG -- : crawler 3 found 57\n" + # "[2020-01-26 16:20:23.805] DEBUG -- : crawler 1 found 58\n" + # "[2020-01-26 16:20:23.805] DEBUG -- : crawler 2 found 59\n" + # "[2020-01-26 16:20:23.806] DEBUG -- : crawler 0 found 5a\n" + # "[2020-01-26 16:20:23.806] DEBUG -- : data-processor 17 got 4e\n" + # "[2020-01-26 16:20:23.806] DEBUG -- : data-processor 18 got 4f\n" + # "[2020-01-26 16:20:23.806] DEBUG -- : data-processor 19 got 50\n" + # "[2020-01-26 16:20:23.807] DEBUG -- : data-processor 1 got 51\n" + # "[2020-01-26 16:20:23.809] DEBUG -- : data-processor 2 got 52\n" + # "[2020-01-26 16:20:23.809] DEBUG -- : data-processor 3 got 53\n" + # "[2020-01-26 16:20:23.809] DEBUG -- : data-processor 0 got 54\n" + # "[2020-01-26 16:20:23.814] DEBUG -- : crawler 1 found 5b\n" + # "[2020-01-26 16:20:23.815] DEBUG -- : crawler 0 found 5c\n" + # "[2020-01-26 16:20:23.815] DEBUG -- : crawler 2 found 5d\n" + # "[2020-01-26 16:20:23.816] DEBUG -- : crawler 3 found 5e\n" + # "[2020-01-26 16:20:23.884] DEBUG -- : data-processor 7 got 55\n" + # "[2020-01-26 16:20:23.885] DEBUG -- : data-processor 4 got 56\n" + # "[2020-01-26 16:20:23.885] DEBUG -- : data-processor 6 got 57\n" + # "[2020-01-26 16:20:23.886] DEBUG -- : crawler 1 found 5f\n" + # "[2020-01-26 16:20:23.886] DEBUG -- : crawler 3 found 60\n" + # "[2020-01-26 16:20:23.886] DEBUG -- : crawler 0 found 61\n" + # "[2020-01-26 16:20:23.887] DEBUG -- : crawler 2 found 62\n" + # "[2020-01-26 16:20:23.887] DEBUG -- : data-processor 5 got 58\n" + # "[2020-01-26 16:20:23.895] DEBUG -- : crawler 1 found 63\n" + # "[2020-01-26 16:20:23.895] DEBUG -- : crawler 3 found 64\n" + # "[2020-01-26 16:20:23.895] DEBUG -- : crawler 0 found 65\n" + # "[2020-01-26 16:20:23.896] DEBUG -- : crawler 2 found 66\n" + # "[2020-01-26 16:20:23.910] DEBUG -- : data-processor 9 got 59\n" + # "[2020-01-26 16:20:23.911] DEBUG -- : data-processor 8 got 5a\n" + # "[2020-01-26 16:20:23.911] DEBUG -- : data-processor 10 got 5b\n" + # "[2020-01-26 16:20:23.912] DEBUG -- : data-processor 11 got 5c\n" + # "[2020-01-26 16:20:23.912] DEBUG -- : data-processor 12 got 5d\n" + # "[2020-01-26 16:20:23.913] DEBUG -- : data-processor 13 got 5e\n" + # "[2020-01-26 16:20:23.913] DEBUG -- : data-processor 14 got 5f\n" + # "[2020-01-26 16:20:23.913] DEBUG -- : data-processor 15 got 60\n" + # "[2020-01-26 16:20:23.914] DEBUG -- : crawler 3 found 67\n" + # "[2020-01-26 16:20:23.914] DEBUG -- : crawler 0 found 68\n" + # "[2020-01-26 16:20:23.914] DEBUG -- : crawler 2 found 69\n" + # "[2020-01-26 16:20:23.915] DEBUG -- : crawler 1 found 6a\n" + # "[2020-01-26 16:20:23.922] DEBUG -- : crawler 3 found 6b\n" + # "[2020-01-26 16:20:23.966] INFO -- : \n" + # "crawlers found: 26, 26, 27, 28\n" + # "data processors consumed: 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 4, 4, 4, 4\n" + # "[2020-01-26 16:20:23.985] DEBUG -- : data-processor 19 got 61\n" + # "[2020-01-26 16:20:23.986] DEBUG -- : data-processor 16 got 62\n" + # "[2020-01-26 16:20:23.987] DEBUG -- : data-processor 1 got 63\n" + # "[2020-01-26 16:20:23.987] DEBUG -- : data-processor 2 got 64\n" + # "[2020-01-26 16:20:23.987] DEBUG -- : crawler 0 found 6c\n" + # "[2020-01-26 16:20:23.988] DEBUG -- : crawler 2 found 6d\n" + # "[2020-01-26 16:20:23.988] DEBUG -- : crawler 1 found 6e\n" + # "[2020-01-26 16:20:23.988] DEBUG -- : crawler 3 found 6f\n" + # "[2020-01-26 16:20:23.997] DEBUG -- : crawler 3 found 70\n" + # "[2020-01-26 16:20:23.997] DEBUG -- : crawler 0 found 71\n" + # "[2020-01-26 16:20:23.998] DEBUG -- : crawler 2 found 72\n" + # "[2020-01-26 16:20:23.998] DEBUG -- : crawler 1 found 73\n" + # "[2020-01-26 16:20:24.010] DEBUG -- : data-processor 18 got 65\n" + # "[2020-01-26 16:20:24.010] DEBUG -- : data-processor 3 got 66\n" + # "[2020-01-26 16:20:24.011] DEBUG -- : data-processor 0 got 67\n" + # "[2020-01-26 16:20:24.012] DEBUG -- : data-processor 17 got 68\n" + # "[2020-01-26 16:20:24.012] DEBUG -- : data-processor 7 got 69\n" + # "[2020-01-26 16:20:24.013] DEBUG -- : data-processor 4 got 6a\n" + # "[2020-01-26 16:20:24.013] DEBUG -- : data-processor 6 got 6b\n" + # "[2020-01-26 16:20:24.015] DEBUG -- : crawler 3 found 74\n" + # "[2020-01-26 16:20:24.016] DEBUG -- : crawler 0 found 75\n" + # "[2020-01-26 16:20:24.016] DEBUG -- : crawler 2 found 76\n" + # "[2020-01-26 16:20:24.017] DEBUG -- : crawler 1 found 77\n" + # "[2020-01-26 16:20:24.017] DEBUG -- : data-processor 5 got 6c\n" + # "[2020-01-26 16:20:24.022] DEBUG -- : crawler 3 found 78\n" + # "[2020-01-26 16:20:24.023] DEBUG -- : crawler 0 found 79\n" + # "[2020-01-26 16:20:24.023] DEBUG -- : crawler 2 found 7a\n" + # "[2020-01-26 16:20:24.088] DEBUG -- : data-processor 9 got 6d\n" + # "[2020-01-26 16:20:24.088] DEBUG -- : data-processor 8 got 6e\n" + # "[2020-01-26 16:20:24.089] DEBUG -- : data-processor 10 got 6f\n" + # "[2020-01-26 16:20:24.089] DEBUG -- : data-processor 11 got 70\n" + # "[2020-01-26 16:20:24.115] DEBUG -- : data-processor 12 got 71\n" + # "[2020-01-26 16:20:24.116] DEBUG -- : data-processor 13 got 72\n" + # "[2020-01-26 16:20:24.116] DEBUG -- : data-processor 14 got 73\n" + # "[2020-01-26 16:20:24.117] DEBUG -- : data-processor 15 got 74\n" + # "[2020-01-26 16:20:24.117] DEBUG -- : crawler 1 found 7b\n" + # "[2020-01-26 16:20:24.117] DEBUG -- : crawler 0 found 7c\n" + # "[2020-01-26 16:20:24.117] DEBUG -- : crawler 2 found 7d\n" + # "[2020-01-26 16:20:24.118] DEBUG -- : crawler 3 found 7e\n" + # "[2020-01-26 16:20:24.118] DEBUG -- : data-processor 2 got 75\n" + # "[2020-01-26 16:20:24.118] DEBUG -- : data-processor 19 got 76\n" + # "[2020-01-26 16:20:24.118] DEBUG -- : data-processor 1 got 77\n" + # "[2020-01-26 16:20:24.118] DEBUG -- : data-processor 16 got 78\n" + # "[2020-01-26 16:20:24.127] DEBUG -- : crawler 1 found 7f\n" + # "[2020-01-26 16:20:24.127] DEBUG -- : crawler 0 found 80\n" + # "[2020-01-26 16:20:24.127] DEBUG -- : crawler 2 found 81\n" + # "[2020-01-26 16:20:24.128] DEBUG -- : crawler 3 found 82\n" + # "[2020-01-26 16:20:24.190] DEBUG -- : data-processor 18 got 79\n" + # "[2020-01-26 16:20:24.191] DEBUG -- : data-processor 0 got 7a\n" + # "[2020-01-26 16:20:24.191] DEBUG -- : data-processor 7 got 7b\n" + # "[2020-01-26 16:20:24.191] DEBUG -- : data-processor 3 got 7c\n" + # "[2020-01-26 16:20:24.192] DEBUG -- : crawler 1 found 83\n" + # "[2020-01-26 16:20:24.192] DEBUG -- : crawler 2 found 84\n" + # "[2020-01-26 16:20:24.192] DEBUG -- : crawler 0 found 85\n" + # "[2020-01-26 16:20:24.193] DEBUG -- : crawler 3 found 86\n" + # "[2020-01-26 16:20:24.201] DEBUG -- : crawler 1 found 87\n" + # "[2020-01-26 16:20:24.202] DEBUG -- : crawler 2 found 88\n" + # "[2020-01-26 16:20:24.202] DEBUG -- : crawler 3 found 89\n" + # "[2020-01-26 16:20:24.202] DEBUG -- : crawler 0 found 8a\n" + # "[2020-01-26 16:20:24.220] DEBUG -- : data-processor 17 got 7d\n" + # "[2020-01-26 16:20:24.220] DEBUG -- : data-processor 6 got 7e\n" + # "[2020-01-26 16:20:24.221] DEBUG -- : data-processor 4 got 7f\n" + # "[2020-01-26 16:20:24.221] DEBUG -- : data-processor 5 got 80\n" + # "[2020-01-26 16:20:24.221] DEBUG -- : data-processor 8 got 81\n" + # "[2020-01-26 16:20:24.222] DEBUG -- : data-processor 11 got 82\n" + # "[2020-01-26 16:20:24.222] DEBUG -- : data-processor 9 got 83\n" + # "[2020-01-26 16:20:24.222] DEBUG -- : data-processor 10 got 84\n" + # "[2020-01-26 16:20:24.222] DEBUG -- : crawler 1 found 8b\n" + # "[2020-01-26 16:20:24.223] DEBUG -- : crawler 2 found 8c\n" + # "[2020-01-26 16:20:24.223] DEBUG -- : crawler 3 found 8d\n" + # "[2020-01-26 16:20:24.223] DEBUG -- : crawler 0 found 8e\n" + # "[2020-01-26 16:20:24.232] DEBUG -- : crawler 1 found 8f\n" + # "[2020-01-26 16:20:24.291] DEBUG -- : data-processor 14 got 85\n" + # "[2020-01-26 16:20:24.292] DEBUG -- : data-processor 13 got 86\n" + # "[2020-01-26 16:20:24.292] DEBUG -- : crawler 1 found 93\n" + # "[2020-01-26 16:20:24.292] DEBUG -- : data-processor 15 got 87\n" + # "[2020-01-26 16:20:24.293] DEBUG -- : data-processor 2 got 88\n" + # "[2020-01-26 16:20:24.293] DEBUG -- : crawler 2 found 90\n" + # "[2020-01-26 16:20:24.293] DEBUG -- : crawler 0 found 91\n" + # "[2020-01-26 16:20:24.294] DEBUG -- : crawler 3 found 92\n" + # "[2020-01-26 16:20:24.304] DEBUG -- : crawler 2 found 94\n" + # "[2020-01-26 16:20:24.304] DEBUG -- : crawler 1 found 95\n" + # "[2020-01-26 16:20:24.305] DEBUG -- : crawler 0 found 96\n" + # "[2020-01-26 16:20:24.319] DEBUG -- : data-processor 12 got 89\n" + # "[2020-01-26 16:20:24.320] DEBUG -- : data-processor 19 got 8a\n" + # "[2020-01-26 16:20:24.321] DEBUG -- : data-processor 1 got 8b\n" + # "[2020-01-26 16:20:24.322] DEBUG -- : data-processor 16 got 8c\n" + # "[2020-01-26 16:20:24.322] DEBUG -- : data-processor 18 got 8d\n" + # "[2020-01-26 16:20:24.322] DEBUG -- : data-processor 0 got 8e\n" + # "[2020-01-26 16:20:24.323] DEBUG -- : data-processor 7 got 8f\n" + # "[2020-01-26 16:20:24.323] DEBUG -- : crawler 3 found 97\n" + # "[2020-01-26 16:20:24.323] DEBUG -- : crawler 2 found 98\n" + # "[2020-01-26 16:20:24.324] DEBUG -- : crawler 0 found 99\n" + # "[2020-01-26 16:20:24.326] DEBUG -- : crawler 1 found 9a\n" + # "[2020-01-26 16:20:24.326] DEBUG -- : data-processor 3 got 90\n" + # "[2020-01-26 16:20:24.332] DEBUG -- : crawler 3 found 9b\n" + # "[2020-01-26 16:20:24.333] DEBUG -- : crawler 2 found 9c\n" + # "[2020-01-26 16:20:24.368] INFO -- : \n" + # "crawlers found: 38, 38, 40, 40\n" + # "data processors consumed: 8, 8, 8, 8, 7, 7, 7, 8, 7, 7, 7, 7, 7, 7, 7, 7, 7, 6, 7, 7\n" + # "[2020-01-26 16:20:24.391] DEBUG -- : data-processor 17 got 91\n" + # "[2020-01-26 16:20:24.392] DEBUG -- : data-processor 4 got 92\n" + # "[2020-01-26 16:20:24.393] DEBUG -- : data-processor 11 got 93\n" + # "[2020-01-26 16:20:24.393] DEBUG -- : data-processor 5 got 94\n" + # "[2020-01-26 16:20:24.393] DEBUG -- : crawler 0 found 9d\n" + # "[2020-01-26 16:20:24.394] DEBUG -- : crawler 1 found 9e\n" + # "[2020-01-26 16:20:24.394] DEBUG -- : crawler 3 found 9f\n" + # "[2020-01-26 16:20:24.394] DEBUG -- : crawler 2 found a0\n" + # "[2020-01-26 16:20:24.403] DEBUG -- : crawler 2 found a1\n" + # "[2020-01-26 16:20:24.403] DEBUG -- : crawler 1 found a2\n" + # "[2020-01-26 16:20:24.403] DEBUG -- : crawler 3 found a3\n" + # "[2020-01-26 16:20:24.404] DEBUG -- : crawler 0 found a4\n" + # "[2020-01-26 16:20:24.419] DEBUG -- : data-processor 10 got 95\n" + # "[2020-01-26 16:20:24.420] DEBUG -- : data-processor 8 got 96\n" + # "[2020-01-26 16:20:24.420] DEBUG -- : data-processor 9 got 97\n" + # "[2020-01-26 16:20:24.422] DEBUG -- : data-processor 6 got 98\n" + # "[2020-01-26 16:20:24.422] DEBUG -- : data-processor 14 got 99\n" + # "[2020-01-26 16:20:24.422] DEBUG -- : data-processor 13 got 9a\n" + # "[2020-01-26 16:20:24.423] DEBUG -- : data-processor 15 got 9b\n" + # "[2020-01-26 16:20:24.423] DEBUG -- : crawler 3 found a5\n" + # "[2020-01-26 16:20:24.423] DEBUG -- : crawler 2 found a6\n" + # "[2020-01-26 16:20:24.423] DEBUG -- : crawler 1 found a7\n" + # "[2020-01-26 16:20:24.424] DEBUG -- : crawler 0 found a8\n" + # "[2020-01-26 16:20:24.426] DEBUG -- : data-processor 2 got 9c\n" + # "[2020-01-26 16:20:24.432] DEBUG -- : crawler 3 found a9\n" + # "[2020-01-26 16:20:24.433] DEBUG -- : crawler 1 found aa\n" + # "[2020-01-26 16:20:24.491] DEBUG -- : data-processor 12 got 9d\n" + # "[2020-01-26 16:20:24.493] DEBUG -- : data-processor 19 got 9e\n" + # "[2020-01-26 16:20:24.496] DEBUG -- : data-processor 1 got 9f\n" + # "[2020-01-26 16:20:24.496] DEBUG -- : data-processor 18 got a0\n" + # "[2020-01-26 16:20:24.522] DEBUG -- : data-processor 16 got a1\n" + # "[2020-01-26 16:20:24.522] DEBUG -- : data-processor 7 got a2\n" + # "[2020-01-26 16:20:24.522] DEBUG -- : data-processor 0 got a3\n" + # "[2020-01-26 16:20:24.522] DEBUG -- : crawler 2 found ab\n" + # "[2020-01-26 16:20:24.523] DEBUG -- : data-processor 3 got a4\n" + # "[2020-01-26 16:20:24.523] DEBUG -- : crawler 0 found ac\n" + # "[2020-01-26 16:20:24.523] DEBUG -- : crawler 3 found ad\n" + # "[2020-01-26 16:20:24.523] DEBUG -- : crawler 1 found ae\n" + # "[2020-01-26 16:20:24.524] DEBUG -- : data-processor 17 got a5\n" + # "[2020-01-26 16:20:24.524] DEBUG -- : data-processor 11 got a6\n" + # "[2020-01-26 16:20:24.526] DEBUG -- : data-processor 4 got a7\n" + # "[2020-01-26 16:20:24.530] DEBUG -- : data-processor 5 got a8\n" + # "[2020-01-26 16:20:24.531] DEBUG -- : crawler 2 found af\n" + # "[2020-01-26 16:20:24.532] DEBUG -- : crawler 3 found b0\n" + # "[2020-01-26 16:20:24.532] DEBUG -- : crawler 1 found b1\n" + # "[2020-01-26 16:20:24.532] DEBUG -- : crawler 0 found b2\n" + # "[2020-01-26 16:20:24.596] DEBUG -- : data-processor 10 got a9\n" + # "[2020-01-26 16:20:24.596] DEBUG -- : crawler 0 found b3\n" + # "[2020-01-26 16:20:24.596] DEBUG -- : crawler 3 found b4\n" + # "[2020-01-26 16:20:24.597] DEBUG -- : crawler 2 found b5\n" + # "[2020-01-26 16:20:24.597] DEBUG -- : crawler 1 found b6\n" + # "[2020-01-26 16:20:24.598] DEBUG -- : data-processor 8 got aa\n" + # "[2020-01-26 16:20:24.600] DEBUG -- : data-processor 9 got ab\n" + # "[2020-01-26 16:20:24.600] DEBUG -- : data-processor 14 got ac\n" + # "[2020-01-26 16:20:24.606] DEBUG -- : crawler 2 found b7\n" + # "[2020-01-26 16:20:24.607] DEBUG -- : crawler 3 found b8\n" + # "[2020-01-26 16:20:24.607] DEBUG -- : crawler 0 found b9\n" + # "[2020-01-26 16:20:24.609] DEBUG -- : crawler 1 found ba\n" + # "[2020-01-26 16:20:24.621] DEBUG -- : data-processor 6 got ad\n" + # "[2020-01-26 16:20:24.622] DEBUG -- : data-processor 13 got ae\n" + # "[2020-01-26 16:20:24.627] DEBUG -- : data-processor 15 got af\n" + # "[2020-01-26 16:20:24.627] DEBUG -- : data-processor 2 got b0\n" + # "[2020-01-26 16:20:24.628] DEBUG -- : data-processor 12 got b1\n" + # "[2020-01-26 16:20:24.628] DEBUG -- : data-processor 19 got b2\n" + # "[2020-01-26 16:20:24.628] DEBUG -- : data-processor 1 got b3\n" + # "[2020-01-26 16:20:24.628] DEBUG -- : crawler 3 found bb\n" + # "[2020-01-26 16:20:24.629] DEBUG -- : crawler 2 found bc\n" + # "[2020-01-26 16:20:24.629] DEBUG -- : crawler 1 found bd\n" + # "[2020-01-26 16:20:24.629] DEBUG -- : crawler 0 found be\n" + # "[2020-01-26 16:20:24.630] DEBUG -- : data-processor 18 got b4\n" + # "[2020-01-26 16:20:24.637] DEBUG -- : crawler 2 found bf\n" + # "[2020-01-26 16:20:24.638] DEBUG -- : crawler 1 found c0\n" + # "[2020-01-26 16:20:24.697] DEBUG -- : data-processor 16 got b5\n" + # "[2020-01-26 16:20:24.701] DEBUG -- : data-processor 7 got b6\n" + # "[2020-01-26 16:20:24.704] DEBUG -- : data-processor 0 got b7\n" + # "[2020-01-26 16:20:24.704] DEBUG -- : data-processor 3 got b8\n" + # "[2020-01-26 16:20:24.704] DEBUG -- : crawler 0 found c1\n" + # "[2020-01-26 16:20:24.704] DEBUG -- : crawler 3 found c2\n" + # "[2020-01-26 16:20:24.705] DEBUG -- : crawler 2 found c3\n" + # "[2020-01-26 16:20:24.705] DEBUG -- : crawler 1 found c4\n" + # "[2020-01-26 16:20:24.714] DEBUG -- : crawler 3 found c5\n" + # "[2020-01-26 16:20:24.715] DEBUG -- : crawler 0 found c6\n" + # "[2020-01-26 16:20:24.715] DEBUG -- : crawler 1 found c7\n" + # "[2020-01-26 16:20:24.715] DEBUG -- : crawler 2 found c8\n" + # "[2020-01-26 16:20:24.727] DEBUG -- : data-processor 11 got b9\n" + # "[2020-01-26 16:20:24.728] DEBUG -- : data-processor 17 got ba\n" + # "[2020-01-26 16:20:24.729] DEBUG -- : data-processor 4 got bb\n" + # "[2020-01-26 16:20:24.729] DEBUG -- : data-processor 5 got bc\n" + # "[2020-01-26 16:20:24.730] DEBUG -- : data-processor 10 got bd\n" + # "[2020-01-26 16:20:24.730] DEBUG -- : data-processor 8 got be\n" + # "[2020-01-26 16:20:24.734] DEBUG -- : data-processor 14 got bf\n" + # "[2020-01-26 16:20:24.734] DEBUG -- : crawler 0 found c9\n" + # "[2020-01-26 16:20:24.734] DEBUG -- : crawler 3 found ca\n" + # "[2020-01-26 16:20:24.735] DEBUG -- : crawler 2 found cb\n" + # "[2020-01-26 16:20:24.736] DEBUG -- : crawler 1 found cc\n" + # "[2020-01-26 16:20:24.737] DEBUG -- : data-processor 9 got c0\n" + # "[2020-01-26 16:20:24.744] DEBUG -- : crawler 1 found cd\n" + # "[2020-01-26 16:20:24.744] DEBUG -- : crawler 3 found ce\n" + # "[2020-01-26 16:20:24.745] DEBUG -- : crawler 2 found cf\n" + # "[2020-01-26 16:20:24.745] DEBUG -- : crawler 0 found d0\n" + # "[2020-01-26 16:20:24.768] INFO -- : \n" + # "crawlers found: 50, 52, 53, 53\n" + # "data processors consumed: 10, 10, 10, 10, 10, 10, 9, 10, 10, 10, 10, 10, 9, 9, 10, 9, 9, 9, 9, 9\n" + # "[2020-01-26 16:20:24.802] DEBUG -- : data-processor 6 got c1\n" + # "[2020-01-26 16:20:24.803] DEBUG -- : data-processor 13 got c2\n" + # "[2020-01-26 16:20:24.805] DEBUG -- : data-processor 2 got c3\n" + # "[2020-01-26 16:20:24.808] DEBUG -- : data-processor 15 got c4\n" + # "[2020-01-26 16:20:24.827] DEBUG -- : data-processor 12 got c5\n" + # "[2020-01-26 16:20:24.835] DEBUG -- : data-processor 1 got c6\n" + # "[2020-01-26 16:20:24.835] DEBUG -- : data-processor 19 got c7\n" + # "[2020-01-26 16:20:24.835] DEBUG -- : crawler 1 found d1\n" + # "[2020-01-26 16:20:24.836] DEBUG -- : data-processor 18 got c8\n" + # "[2020-01-26 16:20:24.836] DEBUG -- : data-processor 7 got c9\n" + # "[2020-01-26 16:20:24.836] DEBUG -- : data-processor 16 got ca\n" + # "[2020-01-26 16:20:24.836] DEBUG -- : crawler 2 found d2\n" + # "[2020-01-26 16:20:24.837] DEBUG -- : crawler 3 found d3\n" + # "[2020-01-26 16:20:24.837] DEBUG -- : crawler 0 found d4\n" + # "[2020-01-26 16:20:24.837] DEBUG -- : data-processor 0 got cb\n" + # "[2020-01-26 16:20:24.838] DEBUG -- : data-processor 3 got cc\n" + # "[2020-01-26 16:20:24.845] DEBUG -- : crawler 1 found d5\n" + # "[2020-01-26 16:20:24.845] DEBUG -- : crawler 2 found d6\n" + # "[2020-01-26 16:20:24.845] DEBUG -- : crawler 0 found d7\n" + # "[2020-01-26 16:20:24.904] DEBUG -- : data-processor 10 got cd\n" + # "[2020-01-26 16:20:24.904] DEBUG -- : data-processor 11 got ce\n" + # "[2020-01-26 16:20:24.904] DEBUG -- : crawler 3 found d8\n" + # "[2020-01-26 16:20:24.905] DEBUG -- : crawler 1 found d9\n" + # "[2020-01-26 16:20:24.905] DEBUG -- : crawler 2 found da\n" + # "[2020-01-26 16:20:24.905] DEBUG -- : crawler 0 found db\n" + # "[2020-01-26 16:20:24.906] DEBUG -- : data-processor 17 got cf\n" + # "[2020-01-26 16:20:24.908] DEBUG -- : data-processor 5 got d0\n" + # "[2020-01-26 16:20:24.914] DEBUG -- : crawler 0 found dc\n" + # "[2020-01-26 16:20:24.915] DEBUG -- : crawler 3 found dd\n" + # "[2020-01-26 16:20:24.915] DEBUG -- : crawler 1 found de\n" + # "[2020-01-26 16:20:24.931] DEBUG -- : data-processor 8 got d1\n" + # "[2020-01-26 16:20:24.933] DEBUG -- : data-processor 4 got d2\n" + # "[2020-01-26 16:20:24.936] DEBUG -- : data-processor 14 got d3\n" + # "[2020-01-26 16:20:24.936] DEBUG -- : data-processor 9 got d4\n" + # "[2020-01-26 16:20:24.936] DEBUG -- : data-processor 13 got d5\n" + # "[2020-01-26 16:20:24.936] DEBUG -- : data-processor 6 got d6\n" + # "[2020-01-26 16:20:24.937] DEBUG -- : crawler 2 found df\n" + # "[2020-01-26 16:20:24.937] DEBUG -- : crawler 0 found e0\n" + # "[2020-01-26 16:20:24.937] DEBUG -- : crawler 1 found e1\n" + # "[2020-01-26 16:20:24.937] DEBUG -- : crawler 3 found e2\n" + # "[2020-01-26 16:20:24.938] DEBUG -- : data-processor 2 got d7\n" + # "[2020-01-26 16:20:24.939] DEBUG -- : data-processor 15 got d8\n" + # "[2020-01-26 16:20:24.948] DEBUG -- : crawler 2 found e3\n" + # "[2020-01-26 16:20:24.948] DEBUG -- : crawler 0 found e4\n" + # "[2020-01-26 16:20:24.948] DEBUG -- : crawler 1 found e5\n" + # "[2020-01-26 16:20:25.006] DEBUG -- : data-processor 12 got d9\n" + # "[2020-01-26 16:20:25.006] DEBUG -- : data-processor 1 got da\n" + # "[2020-01-26 16:20:25.006] DEBUG -- : data-processor 7 got db\n" + # "[2020-01-26 16:20:25.011] DEBUG -- : data-processor 18 got dc\n" + # "[2020-01-26 16:20:25.011] DEBUG -- : crawler 3 found e6\n" + # "[2020-01-26 16:20:25.012] DEBUG -- : crawler 2 found e7\n" + # "[2020-01-26 16:20:25.012] DEBUG -- : crawler 0 found e8\n" + # "[2020-01-26 16:20:25.012] DEBUG -- : crawler 1 found e9\n" + # "[2020-01-26 16:20:25.021] DEBUG -- : crawler 0 found ea\n" + # "[2020-01-26 16:20:25.022] DEBUG -- : crawler 1 found eb\n" + # "[2020-01-26 16:20:25.022] DEBUG -- : crawler 3 found ec\n" + # "[2020-01-26 16:20:25.022] DEBUG -- : crawler 2 found ed\n" + # "[2020-01-26 16:20:25.032] DEBUG -- : data-processor 16 got dd\n" + # "[2020-01-26 16:20:25.033] DEBUG -- : data-processor 0 got de\n" + # "[2020-01-26 16:20:25.035] DEBUG -- : data-processor 19 got df\n" + # "[2020-01-26 16:20:25.036] DEBUG -- : data-processor 3 got e0\n" + # "[2020-01-26 16:20:25.036] DEBUG -- : data-processor 11 got e1\n" + # "[2020-01-26 16:20:25.036] DEBUG -- : data-processor 10 got e2\n" + # "[2020-01-26 16:20:25.040] DEBUG -- : data-processor 17 got e3\n" + # "[2020-01-26 16:20:25.040] DEBUG -- : data-processor 5 got e4\n" + # "[2020-01-26 16:20:25.040] DEBUG -- : crawler 0 found ee\n" + # "[2020-01-26 16:20:25.041] DEBUG -- : crawler 1 found ef\n" + # "[2020-01-26 16:20:25.041] DEBUG -- : crawler 3 found f0\n" + # "[2020-01-26 16:20:25.041] DEBUG -- : crawler 2 found f1\n" + # "[2020-01-26 16:20:25.050] DEBUG -- : crawler 0 found f2\n" + # "[2020-01-26 16:20:25.051] DEBUG -- : crawler 1 found f3\n" + # "[2020-01-26 16:20:25.053] DEBUG -- : crawler 3 found f4\n" + # "[2020-01-26 16:20:25.053] DEBUG -- : crawler 2 found f5\n" + # "[2020-01-26 16:20:25.106] DEBUG -- : data-processor 8 got e5\n" + # "[2020-01-26 16:20:25.106] DEBUG -- : data-processor 4 got e6\n" + # "[2020-01-26 16:20:25.107] DEBUG -- : data-processor 14 got e7\n" + # "[2020-01-26 16:20:25.111] DEBUG -- : data-processor 9 got e8\n" + # "[2020-01-26 16:20:25.132] DEBUG -- : data-processor 13 got e9\n" + # "[2020-01-26 16:20:25.139] DEBUG -- : data-processor 6 got ea\n" + # "[2020-01-26 16:20:25.139] DEBUG -- : data-processor 15 got eb\n" + # "[2020-01-26 16:20:25.140] DEBUG -- : data-processor 2 got ec\n" + # "[2020-01-26 16:20:25.140] DEBUG -- : crawler 0 found f6\n" + # "[2020-01-26 16:20:25.140] DEBUG -- : crawler 1 found f7\n" + # "[2020-01-26 16:20:25.140] DEBUG -- : crawler 2 found f8\n" + # "[2020-01-26 16:20:25.141] DEBUG -- : crawler 3 found f9\n" + # "[2020-01-26 16:20:25.141] DEBUG -- : data-processor 12 got ed\n" + # "[2020-01-26 16:20:25.141] DEBUG -- : data-processor 7 got ee\n" + # "[2020-01-26 16:20:25.141] DEBUG -- : data-processor 1 got ef\n" + # "[2020-01-26 16:20:25.141] DEBUG -- : data-processor 18 got f0\n" + # "[2020-01-26 16:20:25.149] DEBUG -- : crawler 0 found fa\n" + # "[2020-01-26 16:20:25.149] DEBUG -- : crawler 1 found fb\n" + # "[2020-01-26 16:20:25.163] INFO -- : \n" + # "crawlers found: 62, 64, 63, 62\n" + # "data processors consumed: 12, 13, 13, 12, 12, 12, 12, 13, 12, 12, 12, 12, 12, 12, 12, 12, 11, 11, 12, 11\n" + # "[2020-01-26 16:20:25.211] DEBUG -- : data-processor 16 got f1\n" + # "[2020-01-26 16:20:25.250] DEBUG -- : crawler 2 found fc\n" + # "[2020-01-26 16:20:25.250] DEBUG -- : crawler 3 found fd\n" + # "[2020-01-26 16:20:25.260] DEBUG -- : crawler 1 found fe\n" + # "[2020-01-26 16:20:25.260] DEBUG -- : crawler 0 found ff\n"