We should finalize the routes as soon as possible and not wait for the controller to be loaded.

actionpack/lib/action_dispatch/routing/route_set.rb

@@ -533,7 +533,6 @@ module ActionDispatch
       end
 
       def url_for(options)
-        finalize!
         options = (options || {}).reverse_merge!(default_url_options)
 
         handle_positional_args(options)
@@ -559,7 +558,6 @@ module ActionDispatch
       end
 
       def call(env)
-        finalize!
         @router.call(env)
       end
 

railties/lib/rails/application/routes_reloader.rb

@@ -42,7 +42,7 @@ module Rails
 
       def finalize!
         route_sets.each do |routes|
-          ActiveSupport.on_load(:action_controller) { routes.finalize! }
+          routes.finalize!
         end
       end