From bedf235ff68998c0fd87f7531c860f82381312c2 Mon Sep 17 00:00:00 2001 From: Thomas Reynolds Date: Tue, 15 Jul 2014 18:01:45 -0700 Subject: [PATCH] Multiple Source watchers --- Gemfile | 2 + middleman-cli/lib/middleman-cli/server.rb | 4 - .../features/multiple-sources.feature | 27 ++ .../frontmatter-neighbor-app/config.rb | 21 +- .../config.rb | 21 +- .../more-traversal-app/source/layout.erb | 2 +- .../multiple-data-sources-app/config.rb | 3 + .../multiple-data-sources-app/data/data.yml | 1 + .../multiple-data-sources-app/data/two.yml | 1 + .../multiple-data-sources-app/data0/one.yml | 1 + .../multiple-data-sources-app/data1/data1.yml | 1 + .../multiple-data-sources-app/data1/one.yml | 1 + .../multiple-data-sources-app/data2/data2.yml | 1 + .../multiple-data-sources-app/data2/two.yml | 1 + .../source/index.html.erb | 5 + .../fixtures/multiple-sources-app/config.rb | 3 + .../source/index.html.erb | 1 + .../source/override-in-two.html.erb | 1 + .../source0/override-in-one.html.erb | 1 + .../source1/index1.html.erb | 1 + .../source1/override-in-one.html.erb | 1 + .../source2/index2.html.erb | 1 + .../source2/override-in-two.html.erb | 1 + .../fixtures/traversal-app/source/layout.erb | 2 +- .../lib/middleman-core/application.rb | 48 ++- middleman-core/lib/middleman-core/builder.rb | 6 +- .../middleman-core/core_extensions/data.rb | 70 ++-- .../core_extensions/file_watcher.rb | 234 +++---------- .../core_extensions/front_matter.rb | 50 ++- .../middleman-core/core_extensions/i18n.rb | 46 +-- .../middleman-core/core_extensions/routing.rb | 4 +- .../core_extensions/show_exceptions.rb | 2 + .../lib/middleman-core/extensions.rb | 1 + .../extensions/automatic_alt_tags.rb | 9 +- .../extensions/automatic_image_sizes.rb | 9 +- .../lib/middleman-core/file_renderer.rb | 7 +- .../meta_pages/sitemap_resource.rb | 2 +- .../lib/middleman-core/preview_server.rb | 92 ++---- middleman-core/lib/middleman-core/rack.rb | 2 +- .../lib/middleman-core/renderers/less.rb | 10 +- .../lib/middleman-core/renderers/liquid.rb | 12 +- .../lib/middleman-core/renderers/sass.rb | 12 +- .../sitemap/extensions/ignores.rb | 6 +- .../sitemap/extensions/on_disk.rb | 49 +-- .../sitemap/extensions/proxies.rb | 4 +- .../sitemap/extensions/redirects.rb | 2 +- .../sitemap/extensions/request_endpoints.rb | 2 +- .../sitemap/extensions/traversal.rb | 5 +- .../lib/middleman-core/sitemap/resource.rb | 16 +- .../lib/middleman-core/sitemap/store.rb | 13 +- middleman-core/lib/middleman-core/sources.rb | 310 ++++++++++++++++++ .../middleman-core/sources/source_watcher.rb | 268 +++++++++++++++ .../step_definitions/middleman_steps.rb | 12 +- .../step_definitions/server_steps.rb | 4 +- .../lib/middleman-core/template_context.rb | 53 +-- .../lib/middleman-core/template_renderer.rb | 83 ++--- middleman-core/lib/middleman-core/util.rb | 12 +- 57 files changed, 989 insertions(+), 570 deletions(-) create mode 100644 middleman-core/features/multiple-sources.feature create mode 100644 middleman-core/fixtures/multiple-data-sources-app/config.rb create mode 100644 middleman-core/fixtures/multiple-data-sources-app/data/data.yml create mode 100644 middleman-core/fixtures/multiple-data-sources-app/data/two.yml create mode 100644 middleman-core/fixtures/multiple-data-sources-app/data0/one.yml create mode 100644 middleman-core/fixtures/multiple-data-sources-app/data1/data1.yml create mode 100644 middleman-core/fixtures/multiple-data-sources-app/data1/one.yml create mode 100644 middleman-core/fixtures/multiple-data-sources-app/data2/data2.yml create mode 100644 middleman-core/fixtures/multiple-data-sources-app/data2/two.yml create mode 100644 middleman-core/fixtures/multiple-data-sources-app/source/index.html.erb create mode 100644 middleman-core/fixtures/multiple-sources-app/config.rb create mode 100644 middleman-core/fixtures/multiple-sources-app/source/index.html.erb create mode 100644 middleman-core/fixtures/multiple-sources-app/source/override-in-two.html.erb create mode 100644 middleman-core/fixtures/multiple-sources-app/source0/override-in-one.html.erb create mode 100644 middleman-core/fixtures/multiple-sources-app/source1/index1.html.erb create mode 100644 middleman-core/fixtures/multiple-sources-app/source1/override-in-one.html.erb create mode 100644 middleman-core/fixtures/multiple-sources-app/source2/index2.html.erb create mode 100644 middleman-core/fixtures/multiple-sources-app/source2/override-in-two.html.erb create mode 100644 middleman-core/lib/middleman-core/sources.rb create mode 100644 middleman-core/lib/middleman-core/sources/source_watcher.rb diff --git a/Gemfile b/Gemfile index addd4733..7d0c73df 100644 --- a/Gemfile +++ b/Gemfile @@ -6,6 +6,8 @@ gem 'yard', '~> 0.8', require: false # Test tools gem 'pry', '~> 0.10', group: :development, require: false +gem 'pry-debugger', platforms: [:ruby_19, :ruby_20], require: false +gem 'pry-stack_explorer', platforms: [:ruby_19, :ruby_20], require: false gem 'aruba', '~> 0.6', require: false gem 'rspec', '~> 3.0', require: false gem 'fivemat', '~> 1.3', require: false diff --git a/middleman-cli/lib/middleman-cli/server.rb b/middleman-cli/lib/middleman-cli/server.rb index 71d0e779..5c66e3b4 100644 --- a/middleman-cli/lib/middleman-cli/server.rb +++ b/middleman-cli/lib/middleman-cli/server.rb @@ -36,10 +36,6 @@ module Middleman::Cli type: :boolean, default: false, desc: 'Generate profiling report for server startup' - method_option :reload_paths, - type: :string, - default: false, - desc: 'Additional paths to auto-reload when files change' method_option :force_polling, type: :boolean, default: false, diff --git a/middleman-core/features/multiple-sources.feature b/middleman-core/features/multiple-sources.feature new file mode 100644 index 00000000..74b2ede4 --- /dev/null +++ b/middleman-core/features/multiple-sources.feature @@ -0,0 +1,27 @@ +Feature: Allow multiple sources to be setup. + + Scenario: Three source directories. + Given the Server is running at "multiple-sources-app" + When I go to "/index.html" + Then I should see "Default Source" + + When I go to "/index1.html" + Then I should see "Source 1" + + When I go to "/index2.html" + Then I should see "Source 2" + + When I go to "/override-in-two.html" + Then I should see "Overridden 2" + + When I go to "/override-in-one.html" + Then I should see "Opposite 2" + + Scenario: Three data directories. + Given the Server is running at "multiple-data-sources-app" + When I go to "/index.html" + Then I should see "Default: Data Default" + Then I should see "Data 1: Data 1" + Then I should see "Data 2: Data 2" + Then I should see "Override in Two: Overridden 2" + Then I should see "Override in One: Opposite 2" diff --git a/middleman-core/fixtures/frontmatter-neighbor-app/config.rb b/middleman-core/fixtures/frontmatter-neighbor-app/config.rb index 50a368eb..f5edde7f 100644 --- a/middleman-core/fixtures/frontmatter-neighbor-app/config.rb +++ b/middleman-core/fixtures/frontmatter-neighbor-app/config.rb @@ -9,15 +9,18 @@ class NeighborFrontmatter < ::Middleman::Extension resources.each do |resource| next unless resource.source_file - neighbor = "#{resource.source_file}.frontmatter" - if File.exists?(neighbor) - fmdata = app.extensions[:front_matter].frontmatter_and_content(neighbor).first - opts = fmdata.extract!(:layout, :layout_engine, :renderer_options, :directory_index, :content_type) - opts[:renderer_options].symbolize_keys! if opts.key?(:renderer_options) - ignored = fmdata.delete(:ignored) - resource.add_metadata options: opts, page: fmdata - resource.ignore! if ignored == true && !resource.is_a?(::Middleman::Sitemap::ProxyResource) - end + neighbor = "#{resource.source_file[:relative_path]}.frontmatter" + + file = app.files.find(:source, neighbor) + + next unless file + + fmdata = app.extensions[:front_matter].frontmatter_and_content(file[:full_path]).first + opts = fmdata.extract!(:layout, :layout_engine, :renderer_options, :directory_index, :content_type) + opts[:renderer_options].symbolize_keys! if opts.key?(:renderer_options) + ignored = fmdata.delete(:ignored) + resource.add_metadata options: opts, page: fmdata + resource.ignore! if ignored == true && !resource.is_a?(::Middleman::Sitemap::ProxyResource) end end end diff --git a/middleman-core/fixtures/frontmatter-settings-neighbor-app/config.rb b/middleman-core/fixtures/frontmatter-settings-neighbor-app/config.rb index 600b96f8..19a0837b 100644 --- a/middleman-core/fixtures/frontmatter-settings-neighbor-app/config.rb +++ b/middleman-core/fixtures/frontmatter-settings-neighbor-app/config.rb @@ -14,15 +14,18 @@ class NeighborFrontmatter < ::Middleman::Extension resources.each do |resource| next unless resource.source_file - neighbor = "#{resource.source_file}.frontmatter" - if File.exists?(neighbor) - fmdata = app.extensions[:front_matter].frontmatter_and_content(neighbor).first - opts = fmdata.extract!(:layout, :layout_engine, :renderer_options, :directory_index, :content_type) - opts[:renderer_options].symbolize_keys! if opts.key?(:renderer_options) - ignored = fmdata.delete(:ignored) - resource.add_metadata options: opts, page: fmdata - resource.ignore! if ignored == true && !resource.is_a?(::Middleman::Sitemap::ProxyResource) - end + neighbor = "#{resource.source_file[:relative_path]}.frontmatter" + + file = app.files.find(:source, neighbor) + + next unless file + + fmdata = app.extensions[:front_matter].frontmatter_and_content(file[:full_path]).first + opts = fmdata.extract!(:layout, :layout_engine, :renderer_options, :directory_index, :content_type) + opts[:renderer_options].symbolize_keys! if opts.key?(:renderer_options) + ignored = fmdata.delete(:ignored) + resource.add_metadata options: opts, page: fmdata + resource.ignore! if ignored == true && !resource.is_a?(::Middleman::Sitemap::ProxyResource) end end end diff --git a/middleman-core/fixtures/more-traversal-app/source/layout.erb b/middleman-core/fixtures/more-traversal-app/source/layout.erb index 8725e85f..205d5648 100644 --- a/middleman-core/fixtures/more-traversal-app/source/layout.erb +++ b/middleman-core/fixtures/more-traversal-app/source/layout.erb @@ -1,6 +1,6 @@ Path: <%= current_page.path %> -Source: <%= current_page.source_file.sub(root + "/", "") %> +Source: <%= current_page.source_file[:full_path].sub(root + "/", "") %> <% if current_page.parent %> Parent: <%= current_page.parent.path %> diff --git a/middleman-core/fixtures/multiple-data-sources-app/config.rb b/middleman-core/fixtures/multiple-data-sources-app/config.rb new file mode 100644 index 00000000..d3c7a6d0 --- /dev/null +++ b/middleman-core/fixtures/multiple-data-sources-app/config.rb @@ -0,0 +1,3 @@ +files.watch :data, path: File.join(root, 'data0'), priority: 100 +files.watch :data, path: File.join(root, 'data1') +files.watch :data, path: File.join(root, 'data2') \ No newline at end of file diff --git a/middleman-core/fixtures/multiple-data-sources-app/data/data.yml b/middleman-core/fixtures/multiple-data-sources-app/data/data.yml new file mode 100644 index 00000000..a87870c0 --- /dev/null +++ b/middleman-core/fixtures/multiple-data-sources-app/data/data.yml @@ -0,0 +1 @@ +title: Data Default \ No newline at end of file diff --git a/middleman-core/fixtures/multiple-data-sources-app/data/two.yml b/middleman-core/fixtures/multiple-data-sources-app/data/two.yml new file mode 100644 index 00000000..b71ecf18 --- /dev/null +++ b/middleman-core/fixtures/multiple-data-sources-app/data/two.yml @@ -0,0 +1 @@ +title: Overridden Default \ No newline at end of file diff --git a/middleman-core/fixtures/multiple-data-sources-app/data0/one.yml b/middleman-core/fixtures/multiple-data-sources-app/data0/one.yml new file mode 100644 index 00000000..4998511c --- /dev/null +++ b/middleman-core/fixtures/multiple-data-sources-app/data0/one.yml @@ -0,0 +1 @@ +title: Opposite 2 \ No newline at end of file diff --git a/middleman-core/fixtures/multiple-data-sources-app/data1/data1.yml b/middleman-core/fixtures/multiple-data-sources-app/data1/data1.yml new file mode 100644 index 00000000..613a2fe1 --- /dev/null +++ b/middleman-core/fixtures/multiple-data-sources-app/data1/data1.yml @@ -0,0 +1 @@ +title: Data 1 \ No newline at end of file diff --git a/middleman-core/fixtures/multiple-data-sources-app/data1/one.yml b/middleman-core/fixtures/multiple-data-sources-app/data1/one.yml new file mode 100644 index 00000000..ab75d10e --- /dev/null +++ b/middleman-core/fixtures/multiple-data-sources-app/data1/one.yml @@ -0,0 +1 @@ +title: Opposite 1 \ No newline at end of file diff --git a/middleman-core/fixtures/multiple-data-sources-app/data2/data2.yml b/middleman-core/fixtures/multiple-data-sources-app/data2/data2.yml new file mode 100644 index 00000000..489e7f1f --- /dev/null +++ b/middleman-core/fixtures/multiple-data-sources-app/data2/data2.yml @@ -0,0 +1 @@ +title: Data 2 \ No newline at end of file diff --git a/middleman-core/fixtures/multiple-data-sources-app/data2/two.yml b/middleman-core/fixtures/multiple-data-sources-app/data2/two.yml new file mode 100644 index 00000000..79faeaa6 --- /dev/null +++ b/middleman-core/fixtures/multiple-data-sources-app/data2/two.yml @@ -0,0 +1 @@ +title: Overridden 2 \ No newline at end of file diff --git a/middleman-core/fixtures/multiple-data-sources-app/source/index.html.erb b/middleman-core/fixtures/multiple-data-sources-app/source/index.html.erb new file mode 100644 index 00000000..cf0e7a0f --- /dev/null +++ b/middleman-core/fixtures/multiple-data-sources-app/source/index.html.erb @@ -0,0 +1,5 @@ +Default: <%= data.data.title %> +Data 1: <%= data.data1.title %> +Data 2: <%= data.data2.title %> +Override in Two: <%= data.two.title %> +Override in One: <%= data.one.title %> \ No newline at end of file diff --git a/middleman-core/fixtures/multiple-sources-app/config.rb b/middleman-core/fixtures/multiple-sources-app/config.rb new file mode 100644 index 00000000..e48eca88 --- /dev/null +++ b/middleman-core/fixtures/multiple-sources-app/config.rb @@ -0,0 +1,3 @@ +files.watch :source, path: File.join(root, 'source0'), priority: 100 +files.watch :source, path: File.join(root, 'source1') +files.watch :source, path: File.join(root, 'source2') \ No newline at end of file diff --git a/middleman-core/fixtures/multiple-sources-app/source/index.html.erb b/middleman-core/fixtures/multiple-sources-app/source/index.html.erb new file mode 100644 index 00000000..9b273182 --- /dev/null +++ b/middleman-core/fixtures/multiple-sources-app/source/index.html.erb @@ -0,0 +1 @@ +Default Source \ No newline at end of file diff --git a/middleman-core/fixtures/multiple-sources-app/source/override-in-two.html.erb b/middleman-core/fixtures/multiple-sources-app/source/override-in-two.html.erb new file mode 100644 index 00000000..1f817fb2 --- /dev/null +++ b/middleman-core/fixtures/multiple-sources-app/source/override-in-two.html.erb @@ -0,0 +1 @@ +Overridden Default \ No newline at end of file diff --git a/middleman-core/fixtures/multiple-sources-app/source0/override-in-one.html.erb b/middleman-core/fixtures/multiple-sources-app/source0/override-in-one.html.erb new file mode 100644 index 00000000..450b16bd --- /dev/null +++ b/middleman-core/fixtures/multiple-sources-app/source0/override-in-one.html.erb @@ -0,0 +1 @@ +Opposite 2 \ No newline at end of file diff --git a/middleman-core/fixtures/multiple-sources-app/source1/index1.html.erb b/middleman-core/fixtures/multiple-sources-app/source1/index1.html.erb new file mode 100644 index 00000000..25eb5079 --- /dev/null +++ b/middleman-core/fixtures/multiple-sources-app/source1/index1.html.erb @@ -0,0 +1 @@ +Source 1 \ No newline at end of file diff --git a/middleman-core/fixtures/multiple-sources-app/source1/override-in-one.html.erb b/middleman-core/fixtures/multiple-sources-app/source1/override-in-one.html.erb new file mode 100644 index 00000000..f02cb610 --- /dev/null +++ b/middleman-core/fixtures/multiple-sources-app/source1/override-in-one.html.erb @@ -0,0 +1 @@ +Opposite 1 \ No newline at end of file diff --git a/middleman-core/fixtures/multiple-sources-app/source2/index2.html.erb b/middleman-core/fixtures/multiple-sources-app/source2/index2.html.erb new file mode 100644 index 00000000..2afc90fb --- /dev/null +++ b/middleman-core/fixtures/multiple-sources-app/source2/index2.html.erb @@ -0,0 +1 @@ +Source 2 \ No newline at end of file diff --git a/middleman-core/fixtures/multiple-sources-app/source2/override-in-two.html.erb b/middleman-core/fixtures/multiple-sources-app/source2/override-in-two.html.erb new file mode 100644 index 00000000..cb8673bd --- /dev/null +++ b/middleman-core/fixtures/multiple-sources-app/source2/override-in-two.html.erb @@ -0,0 +1 @@ +Overridden 2 \ No newline at end of file diff --git a/middleman-core/fixtures/traversal-app/source/layout.erb b/middleman-core/fixtures/traversal-app/source/layout.erb index 8725e85f..205d5648 100644 --- a/middleman-core/fixtures/traversal-app/source/layout.erb +++ b/middleman-core/fixtures/traversal-app/source/layout.erb @@ -1,6 +1,6 @@ Path: <%= current_page.path %> -Source: <%= current_page.source_file.sub(root + "/", "") %> +Source: <%= current_page.source_file[:full_path].sub(root + "/", "") %> <% if current_page.parent %> Parent: <%= current_page.parent.path %> diff --git a/middleman-core/lib/middleman-core/application.rb b/middleman-core/lib/middleman-core/application.rb index 1121bf23..eefc5dca 100644 --- a/middleman-core/lib/middleman-core/application.rb +++ b/middleman-core/lib/middleman-core/application.rb @@ -73,6 +73,8 @@ module Middleman # Runs after the build is finished define_hook :after_build + define_hook :before_shutdown + define_hook :before_render define_hook :after_render @@ -145,22 +147,19 @@ module Middleman # Setup callbacks which can exclude paths from the sitemap config.define_setting :ignored_sitemap_matchers, { - # dotfiles and folders in the root - root_dotfiles: proc { |file| file.start_with?('.') }, - - # Files starting with an dot, but not .htaccess - source_dotfiles: proc { |file| - file =~ %r{/\.} && file !~ %r{/\.(htaccess|htpasswd|nojekyll)} - }, - # Files starting with an underscore, but not a double-underscore - partials: proc { |file| file =~ %r{/_[^_]} }, + partials: proc { |file| File.basename(file[:relative_path]).match %r{^_[^_]} }, - layout: proc { |file, sitemap_app| - file.start_with?(File.join(sitemap_app.config[:source], 'layout.')) || file.start_with?(File.join(sitemap_app.config[:source], 'layouts/')) + layout: proc { |file, _sitemap_app| + file[:relative_path].to_s.start_with?('layout.') || + file[:relative_path].to_s.start_with?('layouts/') } }, 'Callbacks that can exclude paths from the sitemap' + config.define_setting :watcher_disable, false, 'If the Listen watcher should not run' + config.define_setting :watcher_force_polling, false, 'If the Listen watcher should run in polling mode' + config.define_setting :watcher_latency, nil, 'The Listen watcher latency' + attr_reader :config_context attr_reader :sitemap attr_reader :cache @@ -168,6 +167,7 @@ module Middleman attr_reader :config attr_reader :generic_template_context attr_reader :extensions + attr_reader :sources Contract None => SetOf['Middleman::Application::MiddlewareDescriptor'] attr_reader :middleware @@ -200,7 +200,13 @@ module Middleman @config = ::Middleman::Configuration::ConfigurationManager.new @config.load_settings(self.class.config.all_settings) + config[:source] = ENV['MM_SOURCE'] if ENV['MM_SOURCE'] + @extensions = ::Middleman::ExtensionManager.new(self) + + # Evaluate a passed block if given + config_context.instance_exec(&block) if block_given? + @extensions.auto_activate(:before_sitemap) # Initialize the Sitemap @@ -211,8 +217,6 @@ module Middleman Encoding.default_external = config[:encoding] end - config[:source] = ENV['MM_SOURCE'] if ENV['MM_SOURCE'] - ::Middleman::Extension.clear_after_extension_callbacks @extensions.auto_activate(:before_configuration) @@ -221,7 +225,7 @@ module Middleman run_hook :before_configuration - evaluate_configuration(&block) + evaluate_configuration # This is for making the tests work - since the tests # don't completely reload middleman, I18n.load_path can get @@ -250,10 +254,7 @@ module Middleman @config_context.execute_ready_callbacks end - def evaluate_configuration(&block) - # Evaluate a passed block if given - config_context.instance_exec(&block) if block_given? - + def evaluate_configuration # Check for and evaluate local configuration in `config.rb` local_config = File.join(root, 'config.rb') if File.exist? local_config @@ -296,13 +297,6 @@ module Middleman config[:environment] == key end - # The full path to the source directory - # - # @return [String] - def source_dir - File.join(root, config[:source]) - end - MiddlewareDescriptor = Struct.new(:class, :options, :block) # Use Rack middleware @@ -325,6 +319,10 @@ module Middleman @mappings << MapDescriptor.new(map, block) end + def shutdown! + run_hook :before_shutdown + end + # Work around this bug: http://bugs.ruby-lang.org/issues/4521 # where Ruby will call to_s/inspect while printing exception # messages, which can take a long time (minutes at full CPU) diff --git a/middleman-core/lib/middleman-core/builder.rb b/middleman-core/lib/middleman-core/builder.rb index 23a9ac3a..0af4f471 100644 --- a/middleman-core/lib/middleman-core/builder.rb +++ b/middleman-core/lib/middleman-core/builder.rb @@ -23,7 +23,7 @@ module Middleman # @param [Hash] opts The builder options def initialize(app, opts={}) @app = app - @source_dir = Pathname(@app.source_dir) + @source_dir = Pathname(File.join(@app.root, @app.config[:source])) @build_dir = Pathname(@app.config[:build_dir]) if @build_dir.expand_path.relative_path_from(@source_dir).to_s =~ /\A[.\/]+\Z/ @@ -83,7 +83,7 @@ module Middleman logger.debug '== Checking for Compass sprites' # Double-check for compass sprites - @app.files.find_new_files((@source_dir + @app.config[:images_dir]).relative_path_from(@app.root_path)) + @app.files.find_new_files! @app.sitemap.ensure_resource_list_updated! css_files @@ -170,7 +170,7 @@ module Middleman begin if resource.binary? - export_file!(output_file, Pathname(resource.source_file)) + export_file!(output_file, resource.source_file[:full_path]) else response = @rack.get(URI.escape(resource.request_path)) diff --git a/middleman-core/lib/middleman-core/core_extensions/data.rb b/middleman-core/lib/middleman-core/core_extensions/data.rb index e5824362..e276c2d9 100644 --- a/middleman-core/lib/middleman-core/core_extensions/data.rb +++ b/middleman-core/lib/middleman-core/core_extensions/data.rb @@ -9,26 +9,37 @@ module Middleman class Data < Extension attr_reader :data_store - def before_configuration - @data_store = DataStore.new(app) - app.config.define_setting :data_dir, 'data', 'The directory data files are stored in' - - app.add_to_config_context :data, &method(:data_store) + def initialize(app, config={}, &block) + super # The regex which tells Middleman which files are for data - data_file_matcher = /#{app.config[:data_dir]}\/(.*?)[\w-]+\.(yml|yaml|json)$/ + data_file_matcher = /^(.*?)[\w-]+\.(yml|yaml|json)$/ + + @data_store = DataStore.new(app, data_file_matcher) + app.config.define_setting :data_dir, 'data', 'The directory data files are stored in' + + app.add_to_config_context(:data, &method(:data_store)) + + start_watching(app.config[:data_dir]) + end + + def start_watching(dir) + @original_data_dir = dir + + # Tell the file watcher to observe the :data_dir + @watcher = app.files.watch :data, + path: File.join(app.root, dir), + ignore: proc { |f| !data_file_matcher.match(f[:relative_path]) } # Setup data files before anything else so they are available when # parsing config.rb - app.files.changed(data_file_matcher, &app.extensions[:data].data_store.method(:touch_file)) - app.files.deleted(data_file_matcher, &app.extensions[:data].data_store.method(:remove_file)) + app.files.changed(:data, &@data_store.method(:update_files)) + end - # Tell the file watcher to observe the :data_dir - app.files.watch :data do |path, _app| - path.match data_file_matcher - end + def after_configuration + return unless @original_data_dir != app.config[:data_dir] - app.files.reload_path(app.config[:data_dir]) + @watcher.update_path(app.config[:data_dir]) end helpers do @@ -44,8 +55,9 @@ module Middleman # Setup data store # # @param [Middleman::Application] app The current instance of Middleman - def initialize(app) + def initialize(app, data_file_matcher) @app = app + @data_file_matcher = data_file_matcher @local_data = {} @local_sources = {} @callback_sources = {} @@ -73,22 +85,26 @@ module Middleman @callback_sources end + Contract ArrayOf[IsA['Middleman::SourceFile']], ArrayOf[IsA['Middleman::SourceFile']] => Any + def update_files(updated_files, removed_files) + updated_files.each(&method(:touch_file)) + removed_files.each(&method(:remove_file)) + end + # Update the internal cache for a given file path # # @param [String] file The file to be re-parsed # @return [void] + Contract IsA['Middleman::SourceFile'] => Any def touch_file(file) - root = Pathname(@app.root) - full_path = root + file - extension = File.extname(file) - basename = File.basename(file, extension) - - data_path = full_path.relative_path_from(root + @app.config[:data_dir]) + data_path = file[:relative_path] + extension = File.extname(data_path) + basename = File.basename(data_path, extension) if %w(.yaml .yml).include?(extension) - data = YAML.load_file(full_path) + data = YAML.load_file(file[:full_path]) elsif extension == '.json' - data = ActiveSupport::JSON.decode(full_path.read) + data = ActiveSupport::JSON.decode(file[:full_path].read) else return end @@ -108,13 +124,11 @@ module Middleman # # @param [String] file The file to be cleared # @return [void] + Contract IsA['Middleman::SourceFile'] => Any def remove_file(file) - root = Pathname(@app.root) - full_path = root + file - extension = File.extname(file) - basename = File.basename(file, extension) - - data_path = full_path.relative_path_from(root + @app.config[:data_dir]) + data_path = file[:relative_path] + extension = File.extname(data_path) + basename = File.basename(data_path, extension) data_branch = @local_data diff --git a/middleman-core/lib/middleman-core/core_extensions/file_watcher.rb b/middleman-core/lib/middleman-core/core_extensions/file_watcher.rb index c8be0d89..13cdd3b2 100644 --- a/middleman-core/lib/middleman-core/core_extensions/file_watcher.rb +++ b/middleman-core/lib/middleman-core/core_extensions/file_watcher.rb @@ -1,205 +1,75 @@ -require 'pathname' -require 'set' require 'middleman-core/contracts' +require 'middleman-core/sources' module Middleman module CoreExtensions # API for watching file change events class FileWatcher < Extension - attr_reader :api + # All defined sources. + Contract None => IsA['Middleman::Sources'] + attr_reader :sources + # The default list of ignores. + IGNORES = { + emacs_files: /(^|\/)\.?#/, + tilde_files: /~$/, + ds_store: /\.DS_Store$/, + git: /(^|\/)\.git(ignore|modules|\/)/ + } + + # Setup the extension. def initialize(app, config={}, &block) super + + # Setup source collection. + @sources = ::Middleman::Sources.new(app, + disable_watcher: app.config[:watcher_disable], + force_polling: app.config[:force_polling], + latency: app.config[:watcher_latency]) + + # Add default ignores. + IGNORES.each do |key, value| + @sources.ignore key, :all, value + end + + # Watch current source. + start_watching(app.config[:source]) + + # Expose API to app and config. + app.add_to_instance(:files, &method(:sources)) + app.add_to_config_context(:files, &method(:sources)) end - # Before parsing config, load the data/ directory + # Before we config, find initial files. + # + # @return [void] Contract None => Any def before_configuration - @api = API.new(app) - app.add_to_instance :files, &method(:api) - app.add_to_config_context :files, &method(:api) + @sources.find_new_files! end + # After we config, find new files since config can change paths. + # + # @return [void] Contract None => Any def after_configuration - @api.reload_path('.') - @api.is_ready = true + if @original_source_dir != app.config[:source] + @watcher.update_path(app.config[:source]) + end + + @sources.start! + @sources.find_new_files! end - # Core File Change API class - class API - extend Forwardable - include Contracts + protected - attr_reader :app - attr_reader :known_paths - attr_accessor :is_ready - - def_delegator :@app, :logger - - # Initialize api and internal path cache - def initialize(app) - @app = app - @known_paths = Set.new - @is_ready = false - - @watchers = { - source: proc { |path, _| path.match(/^#{app.config[:source]}\//) }, - library: /^(lib|helpers)\/.*\.rb$/ - } - - @ignores = { - emacs_files: /(^|\/)\.?#/, - tilde_files: /~$/, - ds_store: /\.DS_Store\//, - git: /(^|\/)\.git(ignore|modules|\/)/ - } - - @on_change_callbacks = Set.new - @on_delete_callbacks = Set.new - end - - # Add a proc to watch paths - Contract Symbol, Or[Regexp, Proc] => Any - def watch(name, regex=nil, &block) - @watchers[name] = block_given? ? block : regex - - reload_path('.') if @is_ready - end - - # Add a proc to ignore paths - Contract Symbol, Or[Regexp, Proc] => Any - def ignore(name, regex=nil, &block) - @ignores[name] = block_given? ? block : regex - - reload_path('.') if @is_ready - end - - CallbackDescriptor = Struct.new(:proc, :matcher) - - # Add callback to be run on file change - # - # @param [nil,Regexp] matcher A Regexp to match the change path against - # @return [Array] - Contract Or[Regexp, Proc] => SetOf['Middleman::CoreExtensions::FileWatcher::API::CallbackDescriptor'] - def changed(matcher=nil, &block) - @on_change_callbacks << CallbackDescriptor.new(block, matcher) if block_given? - @on_change_callbacks - end - - # Add callback to be run on file deletion - # - # @param [nil,Regexp] matcher A Regexp to match the deleted path against - # @return [Array] - Contract Or[Regexp, Proc] => SetOf['Middleman::CoreExtensions::FileWatcher::API::CallbackDescriptor'] - def deleted(matcher=nil, &block) - @on_delete_callbacks << CallbackDescriptor.new(block, matcher) if block_given? - @on_delete_callbacks - end - - # Notify callbacks that a file changed - # - # @param [Pathname] path The file that changed - # @return [void] - Contract Or[Pathname, String] => Any - def did_change(path) - path = Pathname(path) - logger.debug "== File Change: #{path}" - @known_paths << path - run_callbacks(path, :changed) - end - - # Notify callbacks that a file was deleted - # - # @param [Pathname] path The file that was deleted - # @return [void] - Contract Or[Pathname, String] => Any - def did_delete(path) - path = Pathname(path) - logger.debug "== File Deletion: #{path}" - @known_paths.delete(path) - run_callbacks(path, :deleted) - end - - # Manually trigger update events - # - # @param [Pathname] path The path to reload - # @param [Boolean] only_new Whether we only look for new files - # @return [void] - Contract Or[String, Pathname], Maybe[Bool] => Any - def reload_path(path, only_new=false) - # chdir into the root directory so Pathname can work with relative paths - Dir.chdir @app.root_path do - path = Pathname(path) - return unless path.exist? - - glob = (path + '**').to_s - subset = @known_paths.select { |p| p.fnmatch(glob) } - - ::Middleman::Util.all_files_under(path, &method(:ignored?)).each do |filepath| - next if only_new && subset.include?(filepath) - - subset.delete(filepath) - did_change(filepath) - end - - subset.each(&method(:did_delete)) unless only_new - end - end - - # Like reload_path, but only triggers events on new files - # - # @param [Pathname] path The path to reload - # @return [void] - Contract Pathname => Any - def find_new_files(path) - reload_path(path, true) - end - - Contract String => Bool - def exists?(path) - p = Pathname(path) - p = p.relative_path_from(Pathname(@app.root)) unless p.relative? - @known_paths.include?(p) - end - - # Whether this path is ignored - # @param [Pathname] path - # @return [Boolean] - Contract Or[String, Pathname] => Bool - def ignored?(path) - path = path.to_s - - watched = @watchers.values.any? { |validator| matches?(validator, path) } - not_ignored = @ignores.values.none? { |validator| matches?(validator, path) } - - !(watched && not_ignored) - end - - Contract Or[Regexp, RespondTo[:call]], String => Bool - def matches?(validator, path) - if validator.is_a? Regexp - !!validator.match(path) - else - !!validator.call(path, @app) - end - end - - protected - - # Notify callbacks for a file given an array of callbacks - # - # @param [Pathname] path The file that was changed - # @param [Symbol] callbacks_name The name of the callbacks method - # @return [void] - Contract Or[Pathname, String], Symbol => Any - def run_callbacks(path, callbacks_name) - path = path.to_s - send(callbacks_name).each do |callback| - next unless callback[:matcher].nil? || path.match(callback[:matcher]) - @app.instance_exec(path, &callback[:proc]) - end - end + # Watch the source directory. + # + # @return [void] + Contract String => Any + def start_watching(dir) + @original_source_dir = dir + @watcher = @sources.watch :source, path: File.join(app.root, dir) end end end diff --git a/middleman-core/lib/middleman-core/core_extensions/front_matter.rb b/middleman-core/lib/middleman-core/core_extensions/front_matter.rb index 4dbb7082..3301badf 100644 --- a/middleman-core/lib/middleman-core/core_extensions/front_matter.rb +++ b/middleman-core/lib/middleman-core/core_extensions/front_matter.rb @@ -27,17 +27,16 @@ module Middleman::CoreExtensions end def before_configuration - app.files.changed(&method(:clear_data)) - app.files.deleted(&method(:clear_data)) + app.files.changed(:source, &method(:clear_data)) end # @return Array Contract ResourceList => ResourceList def manipulate_resource_list(resources) resources.each do |resource| - next if resource.source_file.blank? + next if resource.source_file.nil? - fmdata = data(resource.source_file).first.dup + fmdata = data(resource.source_file[:full_path].to_s).first.dup # Copy over special options # TODO: Should we make people put these under "options" instead of having @@ -61,42 +60,35 @@ module Middleman::CoreExtensions # Get the template data from a path # @param [String] path # @return [String] - Contract String => String + Contract String => Maybe[String] def template_data_for_file(path) data(path).last end Contract String => [Hash, Maybe[String]] def data(path) - p = normalize_path(path) - @cache[p] ||= frontmatter_and_content(p) + file = app.files.find(:source, path) + + return [{}, nil] unless file + + @cache[file[:full_path]] ||= frontmatter_and_content(file[:full_path]) end - def clear_data(file) - # Copied from Sitemap::Store#file_to_path, but without - # removing the file extension - file = File.join(app.root, file) - prefix = app.source_dir.sub(/\/$/, '') + '/' - return unless file.include?(prefix) - path = file.sub(prefix, '') - - @cache.delete(path) + Contract ArrayOf[IsA['Middleman::SourceFile']], ArrayOf[IsA['Middleman::SourceFile']] => Any + def clear_data(updated_files, removed_files) + (updated_files + removed_files).each do |file| + @cache.delete(file[:full_path]) + end end # Get the frontmatter and plain content from a file # @param [String] path # @return [Array] - Contract String => [Hash, Maybe[String]] - def frontmatter_and_content(path) - full_path = if Pathname(path).relative? - File.join(app.source_dir, path) - else - path - end - + Contract Pathname => [Hash, Maybe[String]] + def frontmatter_and_content(full_path) data = {} - return [data, nil] if !app.files.exists?(full_path) || ::Middleman::Util.binary?(full_path) + return [data, nil] if ::Middleman::Util.binary?(full_path) content = File.read(full_path) @@ -121,7 +113,7 @@ module Middleman::CoreExtensions # Parse YAML frontmatter out of a string # @param [String] content # @return [Array] - Contract String, String => Maybe[[Hash, String]] + Contract String, Pathname => Maybe[[Hash, String]] def parse_yaml_front_matter(content, full_path) yaml_regex = /\A(---\s*\n.*?\n?)^(---\s*$\n?)/m if content =~ yaml_regex @@ -146,7 +138,7 @@ module Middleman::CoreExtensions # Parse JSON frontmatter out of a string # @param [String] content # @return [Array] - Contract String, String => Maybe[[Hash, String]] + Contract String, Pathname => Maybe[[Hash, String]] def parse_json_front_matter(content, full_path) json_regex = /\A(;;;\s*\n.*?\n?)^(;;;\s*$\n?)/m @@ -169,9 +161,5 @@ module Middleman::CoreExtensions rescue [{}, content] end - - def normalize_path(path) - path.sub(%r{^#{Regexp.escape(app.source_dir)}\/}, '') - end end end diff --git a/middleman-core/lib/middleman-core/core_extensions/i18n.rb b/middleman-core/lib/middleman-core/core_extensions/i18n.rb index e7d94f69..45459126 100644 --- a/middleman-core/lib/middleman-core/core_extensions/i18n.rb +++ b/middleman-core/lib/middleman-core/core_extensions/i18n.rb @@ -18,28 +18,24 @@ class Middleman::CoreExtensions::Internationalization < ::Middleman::Extension locales_file_path = options[:data] - # Tell the file watcher to observe the :locales_dir - app.files.watch :locales do |path, _app| - path.match(/^#{locales_file_path}\/.*(yml|yaml)$/) - end + # Tell the file watcher to observe the :data_dir + app.files.watch :locales, + path: File.join(app.root, locales_file_path), + ignore: proc { |f| !(/.*(rb|yml|yaml)$/.match(f[:relative_path])) } - app.files.reload_path(locales_file_path) - - @locales_glob = File.join(locales_file_path, '**', '*.{rb,yml,yaml}') - @locales_regex = convert_glob_to_regex(@locales_glob) + # Setup data files before anything else so they are available when + # parsing config.rb + app.files.changed(:locales, &method(:on_file_changed)) @maps = {} @mount_at_root = options[:mount_at_root].nil? ? langs.first : options[:mount_at_root] - configure_i18n - - logger.info "== Locales: #{langs.join(', ')} (Default #{@mount_at_root})" - # Don't output localizable files app.ignore File.join(options[:templates_dir], '**') - app.files.changed(&method(:on_file_changed)) - app.files.deleted(&method(:on_file_changed)) + configure_i18n + + logger.info "== Locales: #{langs.join(', ')} (Default #{@mount_at_root})" end helpers do @@ -87,22 +83,16 @@ class Middleman::CoreExtensions::Internationalization < ::Middleman::Extension private - def on_file_changed(file) - return unless @locales_regex =~ file - + Contract Any, Any => Any + def on_file_changed(_updated_files, _removed_files) @_langs = nil # Clear langs cache + + # TODO, add new file to ::I18n.load_path ::I18n.reload! end - Contract String => Regexp - def convert_glob_to_regex(glob) - # File.fnmatch doesn't support brackets: {rb,yml,yaml} - regex = glob.sub(/\./, '\.').sub(File.join('**', '*'), '.*').sub(/\//, '\/').sub('{rb,yml,yaml}', '(rb|ya?ml)') - %r{^#{regex}} - end - def configure_i18n - ::I18n.load_path += Dir[File.join(app.root, @locales_glob)] + ::I18n.load_path += app.files.by_type(:locales).files.map { |p| p[:full_path].to_s } ::I18n.reload! ::I18n.default_locale = @mount_at_root @@ -116,12 +106,12 @@ class Middleman::CoreExtensions::Internationalization < ::Middleman::Extension if options[:langs] Array(options[:langs]).map(&:to_sym) else - known_langs = app.files.known_paths.select do |p| - p.to_s.match(@locales_regex) && (p.to_s.split(File::SEPARATOR).length == 2) + known_langs = app.files.by_type(:locales).files.select do |p| + p[:relative_path].to_s.split(File::SEPARATOR).length == 1 end known_langs.map { |p| - File.basename(p.to_s).sub(/\.ya?ml$/, '').sub(/\.rb$/, '') + File.basename(p[:relative_path].to_s).sub(/\.ya?ml$/, '').sub(/\.rb$/, '') }.sort.map(&:to_sym) end end diff --git a/middleman-core/lib/middleman-core/core_extensions/routing.rb b/middleman-core/lib/middleman-core/core_extensions/routing.rb index 2bf11eea..9ffc7fb6 100644 --- a/middleman-core/lib/middleman-core/core_extensions/routing.rb +++ b/middleman-core/lib/middleman-core/core_extensions/routing.rb @@ -13,7 +13,7 @@ module Middleman end def before_configuration - app.add_to_config_context :page, &method(:page) + app.add_to_config_context(:page, &method(:page)) end # @return Array @@ -59,7 +59,7 @@ module Middleman if path.is_a?(String) && !path.include?('*') # Normalize path path = Middleman::Util.normalize_path(path) - if path.end_with?('/') || File.directory?(File.join(@app.source_dir, path)) + if path.end_with?('/') || app.files.by_type(:source).watchers.any? { |w| (w.directory + Pathname(path)).directory? } path = File.join(path, @app.config[:index_file]) end end diff --git a/middleman-core/lib/middleman-core/core_extensions/show_exceptions.rb b/middleman-core/lib/middleman-core/core_extensions/show_exceptions.rb index 53c48877..ba9fdade 100644 --- a/middleman-core/lib/middleman-core/core_extensions/show_exceptions.rb +++ b/middleman-core/lib/middleman-core/core_extensions/show_exceptions.rb @@ -6,6 +6,8 @@ module Middleman::CoreExtensions def initialize(app, options_hash={}, &block) super + return if app.config.defines_setting? :show_exceptions + app.config.define_setting :show_exceptions, !!ENV['TEST'], 'Whether to catch and display exceptions' end diff --git a/middleman-core/lib/middleman-core/extensions.rb b/middleman-core/lib/middleman-core/extensions.rb index 1c834587..fa51796d 100644 --- a/middleman-core/lib/middleman-core/extensions.rb +++ b/middleman-core/lib/middleman-core/extensions.rb @@ -10,6 +10,7 @@ module Middleman @auto_activate = { # Activate before the Sitemap is instantiated before_sitemap: Set.new, + # Activate the extension before `config.rb` and the `:before_configuration` hook. before_configuration: Set.new } diff --git a/middleman-core/lib/middleman-core/extensions/automatic_alt_tags.rb b/middleman-core/lib/middleman-core/extensions/automatic_alt_tags.rb index 879cecee..84d23b13 100644 --- a/middleman-core/lib/middleman-core/extensions/automatic_alt_tags.rb +++ b/middleman-core/lib/middleman-core/extensions/automatic_alt_tags.rb @@ -12,13 +12,14 @@ class Middleman::Extensions::AutomaticAltTags < ::Middleman::Extension unless path.include?('://') params[:alt] ||= '' - real_path = path + real_path = path.dup real_path = File.join(images_dir, real_path) unless real_path.start_with?('/') - full_path = File.join(source_dir, real_path) - if File.exist?(full_path) + file = app.files.find(:source, real_path) + + if file && file[:full_path].exist? begin - alt_text = File.basename(full_path, '.*') + alt_text = File.basename(file[:full_path].to_s, '.*') alt_text.capitalize! params[:alt] = alt_text end diff --git a/middleman-core/lib/middleman-core/extensions/automatic_image_sizes.rb b/middleman-core/lib/middleman-core/extensions/automatic_image_sizes.rb index 0f5ef31d..74f3b805 100644 --- a/middleman-core/lib/middleman-core/extensions/automatic_image_sizes.rb +++ b/middleman-core/lib/middleman-core/extensions/automatic_image_sizes.rb @@ -18,13 +18,14 @@ class Middleman::Extensions::AutomaticImageSizes < ::Middleman::Extension if !params.key?(:width) && !params.key?(:height) && !path.include?('://') params[:alt] ||= '' - real_path = path + real_path = path.dup real_path = File.join(config[:images_dir], real_path) unless real_path.start_with?('/') - full_path = File.join(source_dir, real_path) - if File.exist?(full_path) + file = app.files.find(:source, real_path) + + if file && file[:full_path].exist? begin - width, height = ::FastImage.size(full_path, raise_on_failure: true) + width, height = ::FastImage.size(file[:full_path].to_s, raise_on_failure: true) params[:width] = width params[:height] = height rescue FastImage::UnknownImageType diff --git a/middleman-core/lib/middleman-core/file_renderer.rb b/middleman-core/lib/middleman-core/file_renderer.rb index a97eae7c..59d983de 100644 --- a/middleman-core/lib/middleman-core/file_renderer.rb +++ b/middleman-core/lib/middleman-core/file_renderer.rb @@ -100,12 +100,13 @@ module Middleman # Get the template data from a path # @param [String] path # @return [String] - Contract String => String + Contract None => String def template_data_for_file if @app.extensions[:front_matter] - @app.extensions[:front_matter].template_data_for_file(@path) + @app.extensions[:front_matter].template_data_for_file(@path) || '' else - File.read(File.expand_path(@path, source_dir)) + file = @app.files.find(:source, @path) + file.read if file end end diff --git a/middleman-core/lib/middleman-core/meta_pages/sitemap_resource.rb b/middleman-core/lib/middleman-core/meta_pages/sitemap_resource.rb index 27fc03fa..40c4970d 100644 --- a/middleman-core/lib/middleman-core/meta_pages/sitemap_resource.rb +++ b/middleman-core/lib/middleman-core/meta_pages/sitemap_resource.rb @@ -39,7 +39,7 @@ module Middleman build_path = 'Not built' if ignored? props['Build Path'] = build_path if @resource.path != build_path props['URL'] = content_tag(:a, @resource.url, href: @resource.url) unless ignored? - props['Source File'] = @resource.source_file.sub(/^#{Regexp.escape(ENV['MM_ROOT'] + '/')}/, '') + props['Source File'] = @resource.source_file[:full_path].to_s data = @resource.data props['Data'] = data.inspect unless data.empty? diff --git a/middleman-core/lib/middleman-core/preview_server.rb b/middleman-core/lib/middleman-core/preview_server.rb index 82b65acc..646236dc 100644 --- a/middleman-core/lib/middleman-core/preview_server.rb +++ b/middleman-core/lib/middleman-core/preview_server.rb @@ -58,10 +58,6 @@ module Middleman # if the user closed their terminal STDOUT/STDERR won't exist end - if @listener - @listener.stop - @listener = nil - end unmount_instance end @@ -103,6 +99,30 @@ module Middleman app = ::Middleman::Application.new do config[:environment] = opts[:environment].to_sym if opts[:environment] + config[:watcher_disable] = opts[:disable_watcher] + config[:watcher_force_polling] = opts[:force_polling] + config[:watcher_latency] = opts[:latency] + + ready do + match_against = [ + %r{^config\.rb$}, + %r{^environments/[^\.](.*)\.rb$}, + %r{^lib/[^\.](.*)\.rb$}, + %r{^#{@app.config[:helpers_dir]}/[^\.](.*)\.rb$} + ] + + # config.rb + files.watch :reload, + path: root, + ignored: proc { |file| + match_against.none? { |m| file[:relative_path].to_s.match(m) } + } + end + end + + app.files.changed :reload do + $mm_reload = true + @webrick.stop end # Add in the meta pages application @@ -114,41 +134,6 @@ module Middleman app end - def start_file_watcher - return if @listener || @options[:disable_watcher] - - # Watcher Library - require 'listen' - - options = { force_polling: @options[:force_polling] } - options[:latency] = @options[:latency] if @options[:latency] - - @listener = Listen.to(Dir.pwd, options) do |modified, added, removed| - added_and_modified = (modified + added) - - # See if the changed file is config.rb or lib/*.rb - if needs_to_reload?(added_and_modified + removed) - $mm_reload = true - @webrick.stop - else - added_and_modified.each do |path| - relative_path = Pathname(path).relative_path_from(Pathname(Dir.pwd)).to_s - next if app.files.ignored?(relative_path) - app.files.did_change(relative_path) - end - - removed.each do |path| - relative_path = Pathname(path).relative_path_from(Pathname(Dir.pwd)).to_s - next if app.files.ignored?(relative_path) - app.files.did_delete(relative_path) - end - end - end - - # Don't block this thread - @listener.start - end - # Trap some interupt signals and shut down smoothly # @return [void] def register_signal_handlers @@ -196,8 +181,6 @@ module Middleman @webrick ||= setup_webrick(@options[:debug] || false) - start_file_watcher - rack_app = ::Middleman::Rack.new(@app).to_app @webrick.mount '/', ::Rack::Handler::WEBrick, rack_app end @@ -206,33 +189,12 @@ module Middleman # @return [void] def unmount_instance @webrick.unmount '/' + + @app.shutdown! + @app = nil end - # Whether the passed files are config.rb, lib/*.rb or helpers - # @param [Array] paths Array of paths to check - # @return [Boolean] Whether the server needs to reload - def needs_to_reload?(paths) - match_against = [ - %r{^/config\.rb}, - %r{^/environments/[^\.](.*)\.rb$}, - %r{^/lib/[^\.](.*)\.rb$}, - %r{^/#{@app.config[:helpers_dir]}/[^\.](.*)\.rb$} - ] - - if @options[:reload_paths] - @options[:reload_paths].split(',').each do |part| - match_against << %r{^#{part}} - end - end - - paths.any? do |path| - match_against.any? do |matcher| - path.sub(@app.root, '').match matcher - end - end - end - # Returns the URI the preview server will run on # @return [URI] def uri diff --git a/middleman-core/lib/middleman-core/rack.rb b/middleman-core/lib/middleman-core/rack.rb index cd80f61b..6e855ac2 100644 --- a/middleman-core/lib/middleman-core/rack.rb +++ b/middleman-core/lib/middleman-core/rack.rb @@ -123,7 +123,7 @@ module Middleman # Immediately send static file def send_file(resource, env) file = ::Rack::File.new nil - file.path = resource.source_file + file.path = resource.source_file[:full_path] response = file.serving(env) status = response[0] response[1]['Content-Encoding'] = 'gzip' if %w(.svgz .gz).include?(resource.ext) diff --git a/middleman-core/lib/middleman-core/renderers/less.rb b/middleman-core/lib/middleman-core/renderers/less.rb index 5ad79c77..a2f94a0c 100644 --- a/middleman-core/lib/middleman-core/renderers/less.rb +++ b/middleman-core/lib/middleman-core/renderers/less.rb @@ -10,15 +10,17 @@ module Middleman # Default less options app.config.define_setting :less, {}, 'LESS compiler options' - app.after_configuration do - ::Less.paths << File.join(source_dir, config[:css_dir]) - end - # Tell Tilt to use it as well (for inline sass blocks) ::Tilt.register 'less', LocalLoadingLessTemplate ::Tilt.prefer(LocalLoadingLessTemplate) end + def after_configuration + app.files.by_type(:source).watchers.each do |source| + ::Less.paths << (source.directory + app.config[:css_dir]).to_s + end + end + # A SassTemplate for Tilt which outputs debug messages class LocalLoadingLessTemplate < ::Tilt::LessTemplate def prepare diff --git a/middleman-core/lib/middleman-core/renderers/liquid.rb b/middleman-core/lib/middleman-core/renderers/liquid.rb index 78ff5a75..16817caf 100644 --- a/middleman-core/lib/middleman-core/renderers/liquid.rb +++ b/middleman-core/lib/middleman-core/renderers/liquid.rb @@ -7,7 +7,14 @@ module Middleman class Liquid < Middleman::Extension # After config, setup liquid partial paths def after_configuration - ::Liquid::Template.file_system = ::Liquid::LocalFileSystem.new(app.source_dir) + ::Liquid::Template.file_system = self + end + + # Called by Liquid to retrieve a template file + def read_template_file(template_path, _) + file = app.files.find(:source, "_#{template_path}.liquid") + raise ::Liquid::FileSystemError, "No such template '#{template_path}'" unless file + File.read(file[:full_path]) end # @return Array @@ -16,7 +23,8 @@ module Middleman return resources unless app.extensions[:data] resources.each do |resource| - next unless resource.source_file =~ %r{\.liquid$} + next if resource.source_file.nil? + next unless resource.source_file[:relative_path].to_s =~ %r{\.liquid$} # Convert data object into a hash for liquid resource.add_metadata locals: { data: app.extensions[:data].data_store.to_h } diff --git a/middleman-core/lib/middleman-core/renderers/sass.rb b/middleman-core/lib/middleman-core/renderers/sass.rb index 5ec93b9a..a63c8026 100644 --- a/middleman-core/lib/middleman-core/renderers/sass.rb +++ b/middleman-core/lib/middleman-core/renderers/sass.rb @@ -38,6 +38,8 @@ module Middleman def initialize(app, options={}, &block) super + app.files.ignore :sass_cache, :source, /(^|\/)\.sass-cache\// + opts = { output_style: :nested } opts[:line_comments] = false if ENV['TEST'] @@ -59,10 +61,6 @@ module Middleman require 'middleman-core/renderers/sass_functions' end - def before_configuration - app.files.watch :sass_cache, /(^|\/)\.sass-cache\// - end - # A SassTemplate for Tilt which outputs debug messages class SassPlusCSSFilenameTemplate < ::Tilt::SassTemplate def initialize(*args, &block) @@ -111,11 +109,7 @@ module Middleman } if ctx.is_a?(::Middleman::TemplateContext) && file - location_of_sass_file = ctx.source_dir - - parts = basename.split('.') - parts.pop - more_opts[:css_filename] = File.join(location_of_sass_file, ctx.config[:css_dir], parts.join('.')) + more_opts[:css_filename] = file.sub(/\.s[ac]ss$/, '') end options.merge(more_opts) diff --git a/middleman-core/lib/middleman-core/sitemap/extensions/ignores.rb b/middleman-core/lib/middleman-core/sitemap/extensions/ignores.rb index 565c3151..35937e89 100644 --- a/middleman-core/lib/middleman-core/sitemap/extensions/ignores.rb +++ b/middleman-core/lib/middleman-core/sitemap/extensions/ignores.rb @@ -6,13 +6,13 @@ module Middleman def initialize(app, config={}, &block) super - @app.add_to_config_context :ignore, &method(:create_ignore) - @app.define_singleton_method :ignore, &method(:create_ignore) + @app.add_to_config_context(:ignore, &method(:create_ignore)) + @app.define_singleton_method(:ignore, &method(:create_ignore)) # Array of callbacks which can ass ignored @ignored_callbacks = Set.new - @app.sitemap.define_singleton_method :ignored?, &method(:ignored?) + @app.sitemap.define_singleton_method(:ignored?, &method(:ignored?)) end # Ignore a path or add an ignore callback diff --git a/middleman-core/lib/middleman-core/sitemap/extensions/on_disk.rb b/middleman-core/lib/middleman-core/sitemap/extensions/on_disk.rb index 66acb8df..8d319313 100644 --- a/middleman-core/lib/middleman-core/sitemap/extensions/on_disk.rb +++ b/middleman-core/lib/middleman-core/sitemap/extensions/on_disk.rb @@ -24,32 +24,21 @@ module Middleman Contract None => Any def before_configuration - app.files.changed(&method(:touch_file)) - app.files.deleted(&method(:remove_file)) + app.files.changed(:source, &method(:update_files)) + end + + def ignored?(file) + @app.config[:ignored_sitemap_matchers].any? do |_, callback| + callback.call(file, @app) + end end # Update or add an on-disk file path # @param [String] file # @return [void] - Contract String => Any - def touch_file(file) - return false if File.directory?(file) - - begin - @app.sitemap.file_to_path(file) - rescue - return - end - - ignored = @app.config[:ignored_sitemap_matchers].any? do |_, callback| - if callback.arity == 1 - callback.call(file) - else - callback.call(file, @app) - end - end - - @file_paths_on_disk << file unless ignored + Contract ArrayOf[IsA['Middleman::SourceFile']], ArrayOf[IsA['Middleman::SourceFile']] => Any + def update_files(updated_files, removed_files) + return if (updated_files + removed_files).all?(&method(:ignored?)) # Rebuild the sitemap any time a file is touched # in case one of the other manipulators @@ -62,29 +51,19 @@ module Middleman @app.sitemap.ensure_resource_list_updated! unless waiting_for_ready || @app.build? end - # Remove a file from the store - # @param [String] file - # @return [void] - Contract String => Any - def remove_file(file) - return unless @file_paths_on_disk.delete?(file) - - @app.sitemap.rebuild_resource_list!(:removed_file) - - # Force sitemap rebuild so the next request is ready to go. - # Skip this during build because the builder will control sitemap refresh. - @app.sitemap.ensure_resource_list_updated! unless waiting_for_ready || @app.build? + def files_for_sitemap + @app.files.by_type(:source).files.reject(&method(:ignored?)) end # Update the main sitemap resource list # @return Array Contract ResourceList => ResourceList def manipulate_resource_list(resources) - resources + @file_paths_on_disk.map do |file| + resources + files_for_sitemap.map do |file| ::Middleman::Sitemap::Resource.new( @app.sitemap, @app.sitemap.file_to_path(file), - File.join(@app.root, file) + file ) end end diff --git a/middleman-core/lib/middleman-core/sitemap/extensions/proxies.rb b/middleman-core/lib/middleman-core/sitemap/extensions/proxies.rb index cadf81f3..95f30c52 100644 --- a/middleman-core/lib/middleman-core/sitemap/extensions/proxies.rb +++ b/middleman-core/lib/middleman-core/sitemap/extensions/proxies.rb @@ -9,7 +9,7 @@ module Middleman def initialize(app, config={}, &block) super - @app.add_to_config_context :proxy, &method(:create_proxy) + @app.add_to_config_context(:proxy, &method(:create_proxy)) @app.define_singleton_method(:proxy, &method(:create_proxy)) @proxy_configs = Set.new @@ -125,7 +125,7 @@ module Middleman resource end - Contract None => String + Contract None => IsA['Middleman::SourceFile'] def source_file target_resource.source_file end diff --git a/middleman-core/lib/middleman-core/sitemap/extensions/redirects.rb b/middleman-core/lib/middleman-core/sitemap/extensions/redirects.rb index 9b1a4953..174924f1 100644 --- a/middleman-core/lib/middleman-core/sitemap/extensions/redirects.rb +++ b/middleman-core/lib/middleman-core/sitemap/extensions/redirects.rb @@ -10,7 +10,7 @@ module Middleman def initialize(app, config={}, &block) super - @app.add_to_config_context :redirect, &method(:create_redirect) + @app.add_to_config_context(:redirect, &method(:create_redirect)) @redirects = {} end diff --git a/middleman-core/lib/middleman-core/sitemap/extensions/request_endpoints.rb b/middleman-core/lib/middleman-core/sitemap/extensions/request_endpoints.rb index 16a43deb..f0a12dd0 100644 --- a/middleman-core/lib/middleman-core/sitemap/extensions/request_endpoints.rb +++ b/middleman-core/lib/middleman-core/sitemap/extensions/request_endpoints.rb @@ -9,7 +9,7 @@ module Middleman def initialize(app, config={}, &block) super - @app.add_to_config_context :endpoint, &method(:create_endpoint) + @app.add_to_config_context(:endpoint, &method(:create_endpoint)) @endpoints = {} end diff --git a/middleman-core/lib/middleman-core/sitemap/extensions/traversal.rb b/middleman-core/lib/middleman-core/sitemap/extensions/traversal.rb index a1839d76..c86a561b 100644 --- a/middleman-core/lib/middleman-core/sitemap/extensions/traversal.rb +++ b/middleman-core/lib/middleman-core/sitemap/extensions/traversal.rb @@ -76,8 +76,9 @@ module Middleman return true end - full_path = File.join(@app.source_dir, eponymous_directory_path) - File.exist?(full_path) && File.directory?(full_path) + @app.files.by_type(:source).watchers.any? do |source| + (source.directory + Pathname(eponymous_directory_path)).directory? + end end # The path for this resource if it were a directory, and not a file diff --git a/middleman-core/lib/middleman-core/sitemap/resource.rb b/middleman-core/lib/middleman-core/sitemap/resource.rb index 5f9ecc67..ae2ffa73 100644 --- a/middleman-core/lib/middleman-core/sitemap/resource.rb +++ b/middleman-core/lib/middleman-core/sitemap/resource.rb @@ -23,6 +23,7 @@ module Middleman # The on-disk source file for this resource, if there is one # @return [String] + Contract None => Maybe[IsA['Middleman::SourceFile']] attr_reader :source_file # The path to use when requesting this resource. Normally it's @@ -41,6 +42,7 @@ module Middleman # @param [Middleman::Sitemap::Store] store # @param [String] path # @param [String] source_file + Contract IsA['Middleman::Sitemap::Store'], String, Maybe[IsA['Middleman::SourceFile']] => Any def initialize(store, path, source_file=nil) @store = store @app = @store.app @@ -60,7 +62,7 @@ module Middleman Contract None => Bool def template? return false if source_file.nil? - !::Tilt[source_file].nil? + !::Tilt[source_file[:full_path].to_s].nil? end # Merge in new metadata specific to this resource. @@ -108,11 +110,9 @@ module Middleman # @return [String] Contract Hash, Hash => String def render(opts={}, locs={}) - return ::Middleman::FileRenderer.new(@app, source_file).template_data_for_file unless template? + return ::Middleman::FileRenderer.new(@app, source_file[:full_path].to_s).template_data_for_file unless template? - relative_source = Pathname(source_file).relative_path_from(Pathname(@app.root)) - - ::Middleman::Util.instrument 'render.resource', path: relative_source, destination_path: destination_path do + ::Middleman::Util.instrument 'render.resource', path: source_file[:full_path].to_s, destination_path: destination_path do md = metadata opts = md[:options].deep_merge(opts) locs = md[:locals].deep_merge(locs) @@ -123,7 +123,7 @@ module Middleman opts[:layout] = false if %w(.js .json .css .txt).include?(ext) end - renderer = ::Middleman::TemplateRenderer.new(@app, source_file) + renderer = ::Middleman::TemplateRenderer.new(@app, source_file[:full_path].to_s) renderer.render(locs, opts) end end @@ -146,7 +146,7 @@ module Middleman # @return [Boolean] Contract None => Bool def binary? - !source_file.nil? && ::Middleman::Util.binary?(source_file) + !source_file.nil? && ::Middleman::Util.binary?(source_file[:full_path].to_s) end # Ignore a resource directly, without going through the whole @@ -165,7 +165,7 @@ module Middleman # Ignore based on the source path (without template extensions) return true if @app.sitemap.ignored?(path) # This allows files to be ignored by their source file name (with template extensions) - !self.is_a?(ProxyResource) && @app.sitemap.ignored?(source_file.sub("#{@app.source_dir}/", '')) + !self.is_a?(ProxyResource) && @app.sitemap.ignored?(source_file[:relative_path].to_s) end # The preferred MIME content type for this resource based on extension or metadata diff --git a/middleman-core/lib/middleman-core/sitemap/store.rb b/middleman-core/lib/middleman-core/sitemap/store.rb index fd5b6d72..82084c14 100644 --- a/middleman-core/lib/middleman-core/sitemap/store.rb +++ b/middleman-core/lib/middleman-core/sitemap/store.rb @@ -142,21 +142,16 @@ module Middleman # Get the URL path for an on-disk file # @param [String] file # @return [String] - Contract String => String + Contract IsA['Middleman::SourceFile'] => String def file_to_path(file) - file = File.expand_path(file, @app.root) - - prefix = @app.source_dir.sub(/\/$/, '') + '/' - raise "'#{file}' not inside project folder '#{prefix}" unless file.start_with?(prefix) - - path = file.sub(prefix, '') + relative_path = file[:relative_path].to_s # Replace a file name containing automatic_directory_matcher with a folder unless @app.config[:automatic_directory_matcher].nil? - path = path.gsub(@app.config[:automatic_directory_matcher], '/') + relative_path = relative_path.gsub(@app.config[:automatic_directory_matcher], '/') end - extensionless_path(path) + extensionless_path(relative_path) end # Get a path without templating extensions diff --git a/middleman-core/lib/middleman-core/sources.rb b/middleman-core/lib/middleman-core/sources.rb new file mode 100644 index 00000000..1d8729a5 --- /dev/null +++ b/middleman-core/lib/middleman-core/sources.rb @@ -0,0 +1,310 @@ +require 'middleman-core/contracts' + +module Middleman + # The standard "record" that contains information about a file on disk. + SourceFile = Struct.new :relative_path, :full_path, :directory, :type + + # Sources handle multiple on-disk collections of files which make up + # a Middleman project. They are separated by `type` which can then be + # queried. For example, the `source` type represents all content that + # the sitemap uses to build a project. The `data` type represents YAML + # data. The `locales` type represents localization YAML, and so on. + class Sources + extend Forwardable + include Contracts + + # A reference to the current app. + Contract None => IsA['Middleman::Application'] + attr_reader :app + + # Duck-typed definition of a valid source watcher + HANDLER = RespondTo[:changed] + + # Config + Contract None => Hash + attr_reader :options + + # Reference to the global logger. + def_delegator :@app, :logger + + # Built-in types + # :source, :data, :locales, :reload + + # Create a new collection of sources. + # + # @param [Middleman::Application] app The parent app. + # @param [Hash] options Global options. + # @param [Array] watchers Default watchers. + Contract IsA['Middleman::Application'], Maybe[Hash], Maybe[Array] => Any + def initialize(app, options={}, watchers=[]) + @app = app + @watchers = watchers + @sorted_watchers = @watchers.dup.freeze + + @options = options + + # Set of procs wanting to be notified of changes + @on_change_callbacks = [] + + # Global ignores + @ignores = {} + + # Whether we're "running", which means we're in a stable + # watch state after all initialization and config. + @running = false + + @update_count = 0 + @last_update_count = -1 + + # When the app is about to shut down, stop our watchers. + @app.before_shutdown(&method(:stop!)) + end + + # Add a proc to ignore paths with either a regex or block. + # + # @param [Symbol] name A name for the ignore. + # @param [Symbol] type The type of content to apply the ignore to. + # @param [Regexp] regex Ignore by path regex. + # @param [Proc] block Ignore by block evaluation. + # @return [void] + Contract Symbol, Symbol, Or[Regexp, Proc] => Any + def ignore(name, type, regex=nil, &block) + @ignores[name] = { type: type, validator: (block_given? ? block : regex) } + + bump_count + find_new_files! if @running + end + + # Whether this path is ignored. + # + # @param [Middleman::SourceFile] file The file to check. + # @return [Boolean] + Contract SourceFile => Bool + def globally_ignored?(file) + @ignores.values.any? do |descriptor| + ((descriptor[:type] == :all) || (file[:type] == descriptor[:type])) && + matches?(descriptor[:validator], file) + end + end + + # Connect a new watcher. Can either be a type with options, which will + # create a `SourceWatcher` or you can pass in an instantiated class which + # responds to #changed and #deleted + # + # @param [Symbol, #changed, #deleted] type_or_handler The handler. + # @param [Hash] options The watcher options. + # @return [#changed, #deleted] + Contract Or[Symbol, HANDLER], Maybe[Hash] => HANDLER + def watch(type_or_handler, options={}) + handler = if type_or_handler.is_a? Symbol + SourceWatcher.new(self, type_or_handler, options.delete(:path), options) + else + type_or_handler + end + + @watchers << handler + + # The index trick is used so that the sort is stable - watchers with the same priority + # will always be ordered in the same order as they were registered. + n = 0 + @sorted_watchers = @watchers.sort_by do |w| + priority = w.options.fetch(:priority, 50) + n += 1 + [priority, n] + end.reverse.freeze + + handler.changed(&method(:did_change)) + + if @running + handler.poll_once! + handler.listen! + end + + handler + end + + # A list of registered watchers + Contract None => ArrayOf[HANDLER] + def watchers + @sorted_watchers + end + + # Disconnect a specific watcher. + # + # @param [SourceWatcher] watcher The watcher to remove. + # @return [void] + Contract RespondTo[:changed] => Any + def unwatch(watcher) + @watchers.delete(watcher) + + watcher.unwatch + + bump_count + end + + # Filter the collection of watchers by a type. + # + # @param [Symbol] type The watcher type. + # @return [Middleman::Sources] + Contract Symbol => ::Middleman::Sources + def by_type(type) + self.class.new @app, @options, watchers.select { |d| d.type == type } + end + + # Get all files for this collection of watchers. + # + # @return [Array] + Contract None => ArrayOf[SourceFile] + def files + watchers.map(&:files).flatten.uniq { |f| f[:relative_path] } + end + + # Find a file given a type and path. + # + # @param [Symbol] type The file "type". + # @param [String] path The file path. + # @param [Boolean] glob If the path contains wildcard or glob characters. + # @return [Middleman::SourceFile, nil] + Contract Symbol, String, Maybe[Bool] => Maybe[SourceFile] + def find(type, path, glob=false) + watchers + .select { |d| d.type == type } + .map { |d| d.find(path, glob) } + .compact + .first + end + + # Check if a file for a given type exists. + # + # @param [Symbol] type The file "type". + # @param [String] path The file path relative to it's source root. + # @return [Boolean] + Contract Symbol, String => Bool + def exists?(type, path) + watchers + .select { |d| d.type == type } + .any? { |d| d.exists?(path) } + end + + # Check if a file for a given type exists. + # + # @param [Symbol] type The file "type". + # @param [String] path The file path relative to it's source root. + # @return [Boolean] + Contract Symbol, String => Maybe[HANDLER] + def watcher_for_path(type, path) + watchers + .select { |d| d.type == type } + .find { |d| d.exists?(path) } + end + + # Manually poll all watchers for new content. + # + # @return [void] + Contract None => Any + def find_new_files! + return unless @update_count != @last_update_count + + @last_update_count = @update_count + watchers.each(&:poll_once!) + end + + # Start up all listeners. + # + # @return [void] + Contract None => Any + def start! + watchers.each(&:listen!) + @running = true + end + + # Stop the watchers. + # + # @return [void] + Contract None => Any + def stop! + watchers.each(&:stop_listener!) + @running = false + end + + # A callback requires a type and the proc to execute. + CallbackDescriptor = Struct.new :type, :proc + + # Add callback to be run on file change + # + # @param [nil,Regexp] matcher A Regexp to match the change path against + # @return [Set] + Contract Symbol, Proc => ArrayOf[CallbackDescriptor] + def changed(type, &block) + @on_change_callbacks << CallbackDescriptor.new(type, block) + @on_change_callbacks + end + + protected + + # Whether a validator matches a file. + # + # @param [Regexp, #call] validator The match validator. + # @param [Middleman::SourceFile] file The file to check. + # @return [Boolean] + Contract Or[Regexp, RespondTo[:call]], SourceFile => Bool + def matches?(validator, file) + path = file[:relative_path] + if validator.is_a? Regexp + !!validator.match(path.to_s) + else + !!validator.call(path, @app) + end + end + + # Increment the internal counter for changes. + # + # @return [void] + Contract None => Any + def bump_count + @update_count += 1 + end + + # Notify callbacks that a file changed + # + # @param [Middleman::SourceFile] file The file that changed + # @return [void] + Contract ArrayOf[SourceFile], ArrayOf[SourceFile], HANDLER => Any + def did_change(updated_files, removed_files, watcher) + valid_updated = updated_files.select do |file| + watcher_for_path(file[:type], file[:relative_path].to_s) == watcher + end + + valid_removed = removed_files.select do |file| + watcher_for_path(file[:type], file[:relative_path].to_s).nil? + end + + return if valid_updated.empty? && valid_removed.empty? + + bump_count + run_callbacks(@on_change_callbacks, valid_updated, valid_removed) + end + + # Notify callbacks for a file given a set of callbacks + # + # @param [Set] callback_descriptors The registered callbacks. + # @param [Array] files The files that were changed. + # @return [void] + Contract ArrayOf[CallbackDescriptor], ArrayOf[SourceFile], ArrayOf[SourceFile] => Any + def run_callbacks(callback_descriptors, updated_files, removed_files) + callback_descriptors.each do |callback| + if callback[:type] != :all + callback[:proc].call(updated_files, removed_files) + else + valid_updated = updated_files.select { |f| callback[:type] == f[:type] } + valid_removed = removed_files.select { |f| callback[:type] == f[:type] } + + callback[:proc].call(valid_updated, valid_removed) + end + end + end + end +end + +# And, require the actual default implementation for a watcher. +require 'middleman-core/sources/source_watcher.rb' diff --git a/middleman-core/lib/middleman-core/sources/source_watcher.rb b/middleman-core/lib/middleman-core/sources/source_watcher.rb new file mode 100644 index 00000000..e50a2aac --- /dev/null +++ b/middleman-core/lib/middleman-core/sources/source_watcher.rb @@ -0,0 +1,268 @@ +# Watcher Library +require 'listen' + +require 'middleman-core/contracts' + +module Middleman + # The default source watcher implementation. Watches a directory on disk + # and responds to events on changes. + class SourceWatcher + extend Forwardable + include Contracts + + # References to parent `Sources` app and `globally_ignored?` check. + def_delegators :@parent, :app, :globally_ignored? + + # Reference to the singleton logger + def_delegator :app, :logger + + # The type this watcher is representing + Contract None => Symbol + attr_reader :type + + # The directory that is being watched + Contract None => Pathname + attr_reader :directory + + # Options for configuring the watcher + Contract None => Hash + attr_reader :options + + # Construct a new SourceWatcher + # + # @param [Middleman::Sources] parent The parent collection. + # @param [Symbol] type The watcher type. + # @param [String] directory The on-disk path to watch. + # @param [Hash] options Configuration options. + Contract IsA['Middleman::Sources'], Symbol, String, Hash => Any + def initialize(parent, type, directory, options={}) + @parent = parent + @options = options + + @type = type + @directory = Pathname(directory) + + @files = {} + + @validator = options.fetch(:validator, proc { true }) + @ignored = options.fetch(:ignored, proc { false }) + + @disable_watcher = app.build? || @parent.options.fetch(:disable_watcher, false) + @force_polling = @parent.options.fetch(:force_polling, false) + @latency = @parent.options.fetch(:latency, nil) + + @listener = nil + + @on_change_callbacks = Set.new + + @waiting_for_existence = !@directory.exist? + end + + # Change the path of the watcher (if config values upstream change). + # + # @param [String] directory The new path. + # @return [void] + Contract String => Any + def update_path(directory) + @directory = Pathname(directory) + + stop_listener! if @listener + + update([], @files.values) + + poll_once! + + listen! unless @disable_watcher + end + + # Stop watching. + # + # @return [void] + Contract None => Any + def unwatch + stop_listener! + end + + # All the known files in this watcher. + # + # @return [Array] + Contract None => ArrayOf[IsA['Middleman::SourceFile']] + def files + @files.values + end + + # Find a specific file in this watcher. + # + # @param [String, Pathname] path The search path. + # @param [Boolean] glob If the path contains wildcard characters. + # @return [Middleman::SourceFile, nil] + Contract Or[String, Pathname], Maybe[Bool] => Maybe[IsA['Middleman::SourceFile']] + def find(path, glob=false) + p = Pathname(path) + + return nil if p.absolute? && !p.to_s.start_with?(@directory.to_s) + + p = @directory + p if p.relative? + + if glob + found = @files.find { |_, v| v[:relative_path].fnmatch(path) } + found ? found.last : nil + else + @files[p] + end + end + + # Check if a file simply exists in this watcher. + # + # @param [String, Pathname] path The search path. + # @return [Boolean] + Contract Or[String, Pathname] => Bool + def exists?(path) + !find(path).nil? + end + + # Start the `listen` gem Listener. + # + # @return [void] + Contract None => Any + def listen! + return if @disable_watcher || @listener || @waiting_for_existence + + config = { force_polling: @force_polling } + config[:latency] = @latency if @latency + + @listener = ::Listen.to(@directory.to_s, config, &method(:on_listener_change)) + @listener.start + end + + # Stop the listener. + # + # @return [void] + Contract None => Any + def stop_listener! + return unless @listener + + @listener.stop + @listener = nil + end + + # Manually trigger update events. + # + # @return [void] + Contract None => Any + def poll_once! + removed = @files.keys + + updated = [] + + ::Middleman::Util.all_files_under(@directory.to_s).each do |filepath| + removed.delete(filepath) + updated << filepath + end + + update(updated, removed) + + return unless @waiting_for_existence && @directory.exist? + + @waiting_for_existence = false + listen! + end + + # Add callback to be run on file change + # + # @param [Proc] matcher A Regexp to match the change path against + # @return [Set] + Contract Proc => SetOf[Proc] + def changed(&block) + @on_change_callbacks << block + @on_change_callbacks + end + + # Work around this bug: http://bugs.ruby-lang.org/issues/4521 + # where Ruby will call to_s/inspect while printing exception + # messages, which can take a long time (minutes at full CPU) + # if the object is huge or has cyclic references, like this. + def to_s + "#" + end + alias_method :inspect, :to_s # Ruby 2.0 calls inspect for NoMethodError instead of to_s + + protected + + # The `listen` gem callback. + # + # @param [Array] modified List of modified files. + # @param [Array] added List of added files. + # @param [Array] removed List of removed files. + # @return [void] + Contract Array, Array, Array => Any + def on_listener_change(modified, added, removed) + updated = (modified + added) + + return if updated.empty? && removed.empty? + + update(updated.map { |s| Pathname(s) }, removed.map { |s| Pathname(s) }) + end + + # Update our internal list of files on a change. + # + # @param [String, Pathname] path The updated file path. + # @return [void] + Contract ArrayOf[Pathname], ArrayOf[Pathname] => Any + def update(updated_paths, removed_paths) + valid_updates = updated_paths + .map(&method(:path_to_source_file)) + .select(&method(:valid?)) + + valid_updates.each do |f| + @files[f[:full_path]] = f + logger.debug "== Change (#{f[:type]}): #{f[:relative_path]}" + end + + valid_removes = removed_paths + .select(&@files.method(:key?)) + .map(&@files.method(:[])) + .select(&method(:valid?)) + + valid_removes.each do |f| + @files.delete(f[:full_path]) + logger.debug "== Deletion (#{f[:type]}): #{f[:relative_path]}" + end + + run_callbacks(@on_change_callbacks, valid_updates, valid_removes) unless valid_updates.empty? && valid_removes.empty? + end + + # Check if this watcher should care about a file. + # + # @param [Middleman::SourceFile] file The file. + # @return [Boolean] + Contract IsA['Middleman::SourceFile'] => Bool + def valid?(file) + @validator.call(file) && + !globally_ignored?(file) && + !@ignored.call(file) + end + + # Convert a path to a file resprentation. + # + # @param [Pathname] path The path. + # @return [Middleman::SourceFile] + Contract Pathname => IsA['Middleman::SourceFile'] + def path_to_source_file(path) + ::Middleman::SourceFile.new( + path.relative_path_from(@directory), path, @directory, @type) + end + + # Notify callbacks for a file given an array of callbacks + # + # @param [Pathname] path The file that was changed + # @param [Symbol] callbacks_name The name of the callbacks method + # @return [void] + Contract Set, ArrayOf[IsA['Middleman::SourceFile']], ArrayOf[IsA['Middleman::SourceFile']] => Any + def run_callbacks(callbacks, updated_files, removed_files) + callbacks.each do |callback| + callback.call(updated_files, removed_files, self) + end + end + end +end diff --git a/middleman-core/lib/middleman-core/step_definitions/middleman_steps.rb b/middleman-core/lib/middleman-core/step_definitions/middleman_steps.rb index 70dbcf44..cca191f3 100644 --- a/middleman-core/lib/middleman-core/step_definitions/middleman_steps.rb +++ b/middleman-core/lib/middleman-core/step_definitions/middleman_steps.rb @@ -1,17 +1,9 @@ Then /^the file "([^\"]*)" has the contents$/ do |path, contents| write_file(path, contents) - step %Q{the file "#{path}" did change} + @server_inst.files.find_new_files! end Then /^the file "([^\"]*)" is removed$/ do |path| step %Q{I remove the file "#{path}"} - step %Q{the file "#{path}" did delete} -end - -Then /^the file "([^\"]*)" did change$/ do |path| - @server_inst.files.did_change(path) -end - -Then /^the file "([^\"]*)" did delete$/ do |path| - @server_inst.files.did_delete(path) + @server_inst.files.find_new_files! end diff --git a/middleman-core/lib/middleman-core/step_definitions/server_steps.rb b/middleman-core/lib/middleman-core/step_definitions/server_steps.rb index 470a46b1..eccf3331 100644 --- a/middleman-core/lib/middleman-core/step_definitions/server_steps.rb +++ b/middleman-core/lib/middleman-core/step_definitions/server_steps.rb @@ -42,9 +42,11 @@ Given /^the Server is running$/ do ENV['MM_ROOT'] = root_dir initialize_commands = @initialize_commands || [] - initialize_commands.unshift lambda { config[:show_exceptions] = false } @server_inst = ::Middleman::Application.new do + config[:watcher_disable] = true + config[:show_exceptions] = false + initialize_commands.each do |p| instance_exec(&p) end diff --git a/middleman-core/lib/middleman-core/template_context.rb b/middleman-core/lib/middleman-core/template_context.rb index 39c1e9b2..38732719 100644 --- a/middleman-core/lib/middleman-core/template_context.rb +++ b/middleman-core/lib/middleman-core/template_context.rb @@ -23,7 +23,7 @@ module Middleman attr_accessor :current_engine # Shorthand references to global values on the app instance. - def_delegators :@app, :config, :logger, :sitemap, :server?, :build?, :environment?, :data, :extensions, :source_dir, :root + def_delegators :@app, :config, :logger, :sitemap, :server?, :build?, :environment?, :data, :extensions, :root # Initialize a context with the current app and predefined locals and options hashes. # @@ -64,10 +64,10 @@ module Middleman buf_was = save_buffer # Find a layout for this file - layout_path = ::Middleman::TemplateRenderer.locate_layout(@app, layout_name, current_engine) + layout_file = ::Middleman::TemplateRenderer.locate_layout(@app, layout_name, current_engine) # Get the layout engine - extension = File.extname(layout_path) + extension = File.extname(layout_file[:relative_path]) engine = extension[1..-1].to_sym # Store last engine for later (could be inside nested renders) @@ -84,7 +84,7 @@ module Middleman restore_buffer(buf_was) end # Render the layout, with the contents of the block inside. - concat_safe_content render_file(layout_path, @locs, @opts) { content } + concat_safe_content render_file(layout_file, @locs, @opts) { content } ensure # Reset engine back to template's value, regardless of success self.current_engine = engine_was @@ -100,19 +100,19 @@ module Middleman def render(_, name, options={}, &block) name = name.to_s - partial_path = locate_partial(name) + partial_file = locate_partial(name) - raise ::Middleman::TemplateRenderer::TemplateNotFound, "Could not locate partial: #{name}" unless partial_path + raise ::Middleman::TemplateRenderer::TemplateNotFound, "Could not locate partial: #{name}" unless partial_file - r = sitemap.find_resource_by_path(sitemap.file_to_path(partial_path)) + r = sitemap.find_resource_by_path(sitemap.file_to_path(partial_file)) if r && !r.template? - File.read(r.source_file) + File.read(r.source_file[:full_path]) else opts = options.dup locs = opts.delete(:locals) - render_file(partial_path, locs.freeze, opts.freeze, &block) + render_file(partial_file, locs.freeze, opts.freeze, &block) end end @@ -123,43 +123,44 @@ module Middleman # @api private # @param [String] partial_path # @return [String] - Contract String => Maybe[String] + Contract String => Maybe[IsA['Middleman::SourceFile']] def locate_partial(partial_path) return unless resource = sitemap.find_resource_by_path(current_path) # Look for partials relative to the current path - current_dir = File.dirname(resource.source_file) - relative_dir = File.join(current_dir.sub(%r{^#{Regexp.escape(source_dir)}/?}, ''), partial_path) + current_dir = resource.source_file[:relative_path].dirname + non_root = partial_path.to_s.sub(/^\//, '') + relative_dir = current_dir + Pathname(non_root) - partial_path_no_underscore = partial_path.sub(/^_/, '').sub(/\/_/, '/') - relative_dir_no_underscore = File.join(current_dir.sub(%r{^#{Regexp.escape(source_dir)}/?}, ''), partial_path_no_underscore) + non_root_no_underscore = non_root.sub(/^_/, '').sub(/\/_/, '/') + relative_dir_no_underscore = current_dir + Pathname(non_root_no_underscore) - partial = nil + partial_file = nil [ - [relative_dir, { preferred_engine: File.extname(resource.source_file)[1..-1].to_sym }], - [File.join('', partial_path)], - [relative_dir_no_underscore, { try_static: true }], - [File.join('', partial_path_no_underscore), { try_static: true }] + [relative_dir.to_s, { preferred_engine: resource.source_file[:relative_path].extname[1..-1].to_sym }], + [non_root], + [relative_dir_no_underscore.to_s, { try_static: true }], + [non_root_no_underscore, { try_static: true }] ].each do |args| - partial = ::Middleman::TemplateRenderer.resolve_template(@app, *args) - break if partial + partial_file = ::Middleman::TemplateRenderer.resolve_template(@app, *args) + break if partial_file end - partial + partial_file end # Render a path with locs, opts and contents block. # # @api private - # @param [String] path The file path. + # @param [Middleman::SourceFile] file The file. # @param [Hash] locs Template locals. # @param [Hash] opts Template options. # @param [Proc] block A block will be evaluated to return internal contents. # @return [String] The resulting content string. - Contract String, Hash, Hash, Proc => String - def render_file(path, locs, opts, &block) - file_renderer = ::Middleman::FileRenderer.new(@app, path) + Contract IsA['Middleman::SourceFile'], Hash, Hash, Proc => String + def render_file(file, locs, opts, &block) + file_renderer = ::Middleman::FileRenderer.new(@app, file[:relative_path].to_s) file_renderer.render(locs, opts, self, &block) end diff --git a/middleman-core/lib/middleman-core/template_renderer.rb b/middleman-core/lib/middleman-core/template_renderer.rb index bea250d7..050ee207 100644 --- a/middleman-core/lib/middleman-core/template_renderer.rb +++ b/middleman-core/lib/middleman-core/template_renderer.rb @@ -65,8 +65,8 @@ module Middleman end # If we need a layout and have a layout, use it - if layout_path = fetch_layout(engine, options) - layout_renderer = ::Middleman::FileRenderer.new(@app, layout_path) + if layout_file = fetch_layout(engine, options) + layout_renderer = ::Middleman::FileRenderer.new(@app, layout_file[:relative_path].to_s) content = layout_renderer.render(locals, options, context) { content } end @@ -85,11 +85,11 @@ module Middleman # @param [Symbol] engine # @param [Hash] opts # @return [String, Boolean] - Contract Symbol, Hash => Or[String, Bool] + Contract Symbol, Hash => Maybe[IsA['Middleman::SourceFile']] def fetch_layout(engine, opts) # The layout name comes from either the system default or the options local_layout = opts.key?(:layout) ? opts[:layout] : @app.config[:layout] - return false unless local_layout + return unless local_layout # Look for engine-specific options engine_options = @app.config.respond_to?(engine) ? @app.config.send(engine) : {} @@ -108,12 +108,12 @@ module Middleman if local_layout == :_auto_layout # Look for :layout of any extension # If found, use it. If not, continue - locate_layout(:layout, layout_engine) || false + locate_layout(:layout, layout_engine) else # Look for specific layout # If found, use it. If not, error. - if layout_path = locate_layout(local_layout, layout_engine) - layout_path + if layout_file = locate_layout(local_layout, layout_engine) + layout_file else raise ::Middleman::TemplateRenderer::TemplateNotFound, "Could not locate layout: #{local_layout}" end @@ -124,7 +124,7 @@ module Middleman # @param [String] name # @param [Symbol] preferred_engine # @return [String] - Contract Or[String, Symbol], Symbol => Maybe[String] + Contract Or[String, Symbol], Symbol => Maybe[IsA['Middleman::SourceFile']] def locate_layout(name, preferred_engine=nil) self.class.locate_layout(@app, name, preferred_engine) end @@ -133,19 +133,19 @@ module Middleman # @param [String] name # @param [Symbol] preferred_engine # @return [String] - Contract IsA['Middleman::Application'], Or[String, Symbol], Symbol => Maybe[String] + Contract IsA['Middleman::Application'], Or[String, Symbol], Symbol => Maybe[IsA['Middleman::SourceFile']] def self.locate_layout(app, name, preferred_engine=nil) resolve_opts = {} resolve_opts[:preferred_engine] = preferred_engine unless preferred_engine.nil? # Check layouts folder - layout_path = resolve_template(app, File.join(app.config[:layouts_dir], name.to_s), resolve_opts) + layout_file = resolve_template(app, File.join(app.config[:layouts_dir], name.to_s), resolve_opts) # If we didn't find it, check root - layout_path = resolve_template(app, name, resolve_opts) unless layout_path + layout_file = resolve_template(app, name, resolve_opts) unless layout_file # Return the path - layout_path + layout_file end # Find a template on disk given a output path @@ -161,72 +161,53 @@ module Middleman # @param [String] request_path # @option options [Boolean] :preferred_engine If set, try this engine first, then fall back to any engine. # @return [String, Boolean] Either the path to the template, or false - Contract IsA['Middleman::Application'], Or[Symbol, String], Maybe[Hash] => Maybe[String] + Contract IsA['Middleman::Application'], Or[Symbol, String], Maybe[Hash] => Maybe[IsA['Middleman::SourceFile']] def self.resolve_template(app, request_path, options={}) # Find the path by searching or using the cache - request_path = request_path.to_s + relative_path = Util.strip_leading_slash(request_path.to_s) # Cache lookups in build mode only if app.build? - cache.fetch(:resolve_template, request_path, options) do - uncached_resolve_template(app, request_path, options) + cache.fetch(:resolve_template, relative_path, options) do + uncached_resolve_template(app, relative_path, options) end else - uncached_resolve_template(app, request_path, options) + uncached_resolve_template(app, relative_path, options) end end - Contract IsA['Middleman::Application'], String, Hash => Maybe[String] - def self.uncached_resolve_template(app, request_path, options) - relative_path = Util.strip_leading_slash(request_path) - on_disk_path = File.expand_path(relative_path, app.source_dir) - + Contract IsA['Middleman::Application'], Or[Symbol, String], Hash => Maybe[IsA['Middleman::SourceFile']] + def self.uncached_resolve_template(app, relative_path, options) # By default, any engine will do - preferred_engines = ['*'] - preferred_engines << nil if options[:try_static] + preferred_engines = [] # If we're specifically looking for a preferred engine if options.key?(:preferred_engine) extension_class = ::Tilt[options[:preferred_engine]] # Get a list of extensions for a preferred engine - matched_exts = ::Tilt.mappings.select do |_, engines| + preferred_engines += ::Tilt.mappings.select do |_, engines| engines.include? extension_class end.keys - - # Prefer to look for the matched extensions - unless matched_exts.empty? - preferred_engines.unshift('{' + matched_exts.join(',') + '}') - end end - search_paths = preferred_engines.map do |preferred_engine| - path_with_ext = on_disk_path.dup + preferred_engines << '*' + preferred_engines << nil if options[:try_static] + + found_template = nil + + preferred_engines.each do |preferred_engine| + path_with_ext = relative_path.dup path_with_ext << ('.' + preferred_engine) unless preferred_engine.nil? - path_with_ext - end - found_path = nil - search_paths.each do |path_with_ext| - found_path = Dir[path_with_ext].find do |path| - ::Tilt[path] - end + file = app.files.find(:source, path_with_ext, preferred_engine == '*') - unless found_path - found_path = path_with_ext if File.exist?(path_with_ext) - end - - break if found_path + found_template = file if file && (preferred_engine.nil? || ::Tilt[file[:full_path]]) + break if found_template end # If we found one, return it - if found_path - found_path - elsif File.exist?(on_disk_path) - on_disk_path - else - nil - end + found_template end end end diff --git a/middleman-core/lib/middleman-core/util.rb b/middleman-core/lib/middleman-core/util.rb index eabd3e8e..53a183ee 100644 --- a/middleman-core/lib/middleman-core/util.rb +++ b/middleman-core/lib/middleman-core/util.rb @@ -24,9 +24,10 @@ module Middleman # # @param [String] filename The file to check. # @return [Boolean] - Contract String => Bool + Contract Or[String, Pathname] => Bool def binary?(filename) - ext = File.extname(filename) + path = Pathname(filename) + ext = path.extname # We hardcode detecting of gzipped SVG files return true if ext == '.svgz' @@ -38,7 +39,7 @@ module Middleman if mime = ::Rack::Mime.mime_type(dot_ext, nil) !nonbinary_mime?(mime) else - file_contents_include_binary_bytes?(filename) + file_contents_include_binary_bytes?(path.to_s) end end @@ -74,14 +75,15 @@ module Middleman # @private # @param [Hash] data Normal hash # @return [Middleman::Util::HashWithIndifferentAccess] - Contract Maybe[Or[Array, Hash, HashWithIndifferentAccess]] => Maybe[Frozen[Or[HashWithIndifferentAccess, Array]]] + FrozenDataStructure = Frozen[Or[HashWithIndifferentAccess, Array]] + Contract Maybe[Or[Array, Hash, HashWithIndifferentAccess]] => Maybe[FrozenDataStructure] def recursively_enhance(data) if data.is_a? HashWithIndifferentAccess data elsif data.is_a? Hash HashWithIndifferentAccess.new(data) elsif data.is_a? Array - data.map(&method(:recursively_enhance)) + data.map(&method(:recursively_enhance)).freeze else nil end