diff --git a/CHANGELOG.md b/CHANGELOG.md index 598c57ac..685a89f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ master === +* Incremental builds: `--track-dependencies` and `--only-changed` flags (#2220) * Remove Rack support in favor of `resource.filters << proc { |oldbody| newbody }` * `manipulate_resource_list_container!` as a faster, less functional approach. diff --git a/Gemfile.lock b/Gemfile.lock index 3ad30134..15430724 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,13 +1,13 @@ PATH remote: middleman-cli specs: - middleman-cli (4.3.0.rc.4) + middleman-cli (5.0.0.rc.1) thor (>= 0.17.0, < 2.0) PATH remote: middleman-core specs: - middleman-core (4.3.0.rc.4) + middleman-core (5.0.0.rc.1) activesupport (>= 4.2, < 5.2) addressable (~> 2.3) backports (~> 3.11) diff --git a/LICENSE.md b/LICENSE.md index a506b74f..faa6d6c7 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,4 +1,4 @@ -Copyright (c) 2010-2017 Thomas Reynolds +Copyright (c) 2010-2018 Thomas Reynolds Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/middleman-cli/lib/middleman-cli/build.rb b/middleman-cli/lib/middleman-cli/build.rb index 41253290..350a75c9 100644 --- a/middleman-cli/lib/middleman-cli/build.rb +++ b/middleman-cli/lib/middleman-cli/build.rb @@ -36,6 +36,18 @@ module Middleman::Cli type: :boolean, default: false, desc: 'Generate profiling report for the build' + class_option :track_dependencies, + type: :boolean, + default: false, + desc: 'Track file dependencies' + class_option :only_changed, + type: :boolean, + default: false, + desc: 'Only build changed files' + class_option :missing_and_changed, + type: :boolean, + default: false, + desc: 'Only build changed files or files missing from build folder' Middleman::Cli.import_config(self) @@ -71,7 +83,10 @@ module Middleman::Cli builder = Middleman::Builder.new(@app, glob: options['glob'], clean: options['clean'], - parallel: options['parallel']) + parallel: options['parallel'], + only_changed: options['only_changed'], + missing_and_changed: options['missing_and_changed'], + track_dependencies: options['track_dependencies']) builder.thor = self builder.on_build_event(&method(:on_event)) end @@ -108,6 +123,8 @@ module Middleman::Cli say_status :create, target, :green when :identical say_status :identical, target, :blue + when :skipped + say_status :skipped, target, :blue when :updated say_status :updated, target, :yellow else diff --git a/middleman-core/features/extension_hooks.feature b/middleman-core/features/extension_hooks.feature index ae89dfec..12e0eacc 100644 --- a/middleman-core/features/extension_hooks.feature +++ b/middleman-core/features/extension_hooks.feature @@ -5,6 +5,4 @@ Feature: Extension author could use some hooks And the output should contain "/// after_configuration ///" And the output should contain "/// ready ///" And the output should contain "/// before_build ///" - And the output should contain "/// before_render ///" - And the output should contain "/// after_render ///" And the output should contain "/// after_build ///" diff --git a/middleman-core/fixtures/extension-hooks-app/config.rb b/middleman-core/fixtures/extension-hooks-app/config.rb index d0769b81..4a81dda3 100644 --- a/middleman-core/fixtures/extension-hooks-app/config.rb +++ b/middleman-core/fixtures/extension-hooks-app/config.rb @@ -12,14 +12,6 @@ class MyFeature < Middleman::Extension puts '/// ready ///' end - app.before_render do |_body, _path, _locs, _template_class| - puts '/// before_render ///' - end - - app.after_render do |_content, _path, _locs, _template_class| - puts '/// after_render ///' - end - app.before_build do |_builder| puts '/// before_build ///' end diff --git a/middleman-core/fixtures/more-preview-app/source/stylesheets/_below_partial.sass b/middleman-core/fixtures/more-preview-app/source/stylesheets/_below_partial.sass new file mode 100644 index 00000000..f3b95765 --- /dev/null +++ b/middleman-core/fixtures/more-preview-app/source/stylesheets/_below_partial.sass @@ -0,0 +1,2 @@ +footer + font-size: 1px \ No newline at end of file diff --git a/middleman-core/fixtures/more-preview-app/source/stylesheets/_partial.sass b/middleman-core/fixtures/more-preview-app/source/stylesheets/_partial.sass index 3b0e67db..2b171ed9 100644 --- a/middleman-core/fixtures/more-preview-app/source/stylesheets/_partial.sass +++ b/middleman-core/fixtures/more-preview-app/source/stylesheets/_partial.sass @@ -1,2 +1,4 @@ +@import below_partial.sass + body font-size: 18px \ No newline at end of file diff --git a/middleman-core/fixtures/more-preview-app/source/stylesheets/also_needs_partial.css.scss b/middleman-core/fixtures/more-preview-app/source/stylesheets/also_needs_partial.css.scss new file mode 100644 index 00000000..139fe81a --- /dev/null +++ b/middleman-core/fixtures/more-preview-app/source/stylesheets/also_needs_partial.css.scss @@ -0,0 +1,5 @@ +@import "partial.sass"; + +blue { + color: red; +} \ No newline at end of file diff --git a/middleman-core/lib/middleman-core/application.rb b/middleman-core/lib/middleman-core/application.rb index 8484d721..3e8191a6 100644 --- a/middleman-core/lib/middleman-core/application.rb +++ b/middleman-core/lib/middleman-core/application.rb @@ -237,8 +237,6 @@ module Middleman :after_build, :before_shutdown, :before, # Before Rack requests - :before_render, - :after_render, :before_server, :reload ]) diff --git a/middleman-core/lib/middleman-core/builder.rb b/middleman-core/lib/middleman-core/builder.rb index 1316fbc0..eeceaf8e 100644 --- a/middleman-core/lib/middleman-core/builder.rb +++ b/middleman-core/lib/middleman-core/builder.rb @@ -2,6 +2,7 @@ require 'pathname' require 'fileutils' require 'tempfile' require 'parallel' +require 'middleman-core/dependencies' require 'middleman-core/callback_manager' require 'middleman-core/contracts' @@ -33,8 +34,11 @@ module Middleman raise ":build_dir (#{@build_dir}) cannot be a parent of :source_dir (#{@source_dir})" if /\A[.\/]+\Z/.match?(@build_dir.expand_path.relative_path_from(@source_dir).to_s) @glob = options_hash.fetch(:glob) - @cleaning = options_hash.fetch(:clean) @parallel = options_hash.fetch(:parallel, true) + @only_changed = options_hash.fetch(:only_changed, false) + @missing_and_changed = !@only_changed && options_hash.fetch(:missing_and_changed, false) + @track_dependencies = @only_changed || @missing_and_changed || options_hash.fetch(:track_dependencies, false) + @cleaning = options_hash.fetch(:clean) @callbacks = ::Middleman::CallbackManager.new @callbacks.install_methods!(self, [:on_build_event]) @@ -47,6 +51,24 @@ module Middleman @has_error = false @events = {} + if @track_dependencies + begin + @graph = ::Middleman::Dependencies.load_and_deserialize(@app) + rescue ::Middleman::Dependencies::InvalidDepsYAML + logger.error 'dep.yml was corrupt. Dependency graph must be rebuilt.' + @graph = ::Middleman::Dependencies::Graph.new + @only_changed = @missing_and_changed = false + rescue ::Middleman::Dependencies::InvalidatedRubyFiles => e + changed = e.invalidated.map { |f| f[:file] }.join(', ') + logger.error "Some ruby files (#{changed}) have changed since last run. Dependency graph must be rebuilt." + + @graph = ::Middleman::Dependencies::Graph.new + @only_changed = @missing_and_changed = false + end + end + + @invalidated_files = @graph.invalidated if @only_changed || @missing_and_changed + ::Middleman::Util.instrument 'builder.before' do @app.execute_callbacks(:before_build, [self]) end @@ -56,18 +78,34 @@ module Middleman end ::Middleman::Util.instrument 'builder.prerender' do - prerender_css + prerender_css.tap do |resources| + if @track_dependencies + resources.each do |r| + dependency = r[1] + @graph.add_dependency(dependency) unless dependency.nil? + end + end + end end ::Middleman::Profiling.start ::Middleman::Util.instrument 'builder.output' do - output_files + output_files.tap do |resources| + if @track_dependencies + resources.each do |r| + dependency = r[1] + @graph.add_dependency(dependency) unless dependency.nil? + end + end + end end ::Middleman::Profiling.report('build') unless @has_error + ::Middleman::Dependencies.serialize_and_save(@app, @graph) if @track_dependencies + ::Middleman::Util.instrument 'builder.clean' do clean! if @cleaning end @@ -82,12 +120,14 @@ module Middleman # Pre-request CSS to give Compass a chance to build sprites # @return [Array] List of css resources that were output. - Contract ResourceList + Contract ArrayOf[[Pathname, Maybe[::Middleman::Dependencies::Dependency]]] def prerender_css logger.debug '== Prerendering CSS' + resources = @app.sitemap.by_extension('.css').to_a + css_files = ::Middleman::Util.instrument 'builder.prerender.output' do - output_resources(@app.sitemap.by_extension('.css').to_a) + output_resources(resources) end ::Middleman::Util.instrument 'builder.prerender.check-files' do @@ -103,7 +143,7 @@ module Middleman # Find all the files we need to output and do so. # @return [Array] List of resources that were output. - Contract OldResourceList + Contract ArrayOf[[Pathname, Maybe[::Middleman::Dependencies::Dependency]]] def output_files logger.debug '== Building files' @@ -112,20 +152,10 @@ module Middleman resources = non_css_resources .sort_by { |resource| SORT_ORDER.index(resource.ext) || 100 } - if @glob - resources = resources.select do |resource| - if defined?(::File::FNM_EXTGLOB) - File.fnmatch(@glob, resource.destination_path, ::File::FNM_EXTGLOB) - else - File.fnmatch(@glob, resource.destination_path) - end - end - end - output_resources(resources.to_a) end - Contract OldResourceList => OldResourceList + Contract OldResourceList => ArrayOf[Or[Bool, [Pathname, Maybe[::Middleman::Dependencies::Dependency]]]] def output_resources(resources) res_count = resources.count @@ -156,16 +186,19 @@ module Middleman resources[r].map!(&method(:output_resource)) end - outputs.flatten! + outputs.flatten!(1) outputs else resources.map(&method(:output_resource)) end - @has_error = true if results.any? { |r| r == false } + without_errors = results.reject { |r| r == false } + + @has_error = results.size > without_errors.size if @cleaning && !@has_error - results.each do |p| + results.each do |r| + p = r[0] next unless p.exist? # handle UTF-8-MAC filename on MacOS @@ -179,19 +212,21 @@ module Middleman end end - resources + without_errors end # Figure out the correct event mode. # @param [Pathname] output_file The output file path. # @param [String] source The source file path. # @return [Symbol] - Contract Pathname, String => Symbol - def which_mode(output_file, source) + Contract Pathname, String, Bool => Symbol + def which_mode(output_file, source, binary) if !output_file.exist? :created + elsif FileUtils.compare_file(source.to_s, output_file.to_s) + binary ? :skipped : :identical else - FileUtils.compare_file(source.to_s, output_file.to_s) ? :identical : :updated + :updated end end @@ -216,8 +251,8 @@ module Middleman # @param [Pathname] output_file The path to output to. # @param [String|Pathname] source The source path or contents. # @return [void] - Contract Pathname, Or[String, Pathname] => Any - def export_file!(output_file, source) + Contract Pathname, Or[String, Pathname], Maybe[Bool] => Any + def export_file!(output_file, source, binary = false) ::Middleman::Util.instrument 'write_file', output_file: output_file do source = write_tempfile(output_file, source.to_s) if source.is_a? String @@ -227,9 +262,9 @@ module Middleman [::FileUtils.method(:cp), source.to_s] end - mode = which_mode(output_file, source_path) + mode = which_mode(output_file, source_path, binary) - if %i[created updated].include? mode + if mode == :created ::FileUtils.mkdir_p(output_file.dirname) method.call(source_path, output_file.to_s) end @@ -243,23 +278,57 @@ module Middleman # Try to output a resource and capture errors. # @param [Middleman::Sitemap::Resource] resource The resource. # @return [void] - Contract IsA['Middleman::Sitemap::Resource'] => Or[Pathname, Bool] + Contract IsA['Middleman::Sitemap::Resource'] => Or[Bool, [Pathname, Maybe[::Middleman::Dependencies::Dependency]]] def output_resource(resource) ::Middleman::Util.instrument 'builder.output.resource', path: File.basename(resource.destination_path) do - output_file = @build_dir + resource.destination_path.gsub('%20', ' ') - begin + output_file = @build_dir + resource.destination_path.gsub('%20', ' ') + + if @track_dependencies && (@only_changed || @missing_and_changed) + path = resource.file_descriptor[:full_path].to_s + + if @only_changed && !@invalidated_files.include?(path) + trigger(:skipped, output_file) + return [output_file, nil] + elsif @missing_and_changed && File.exist?(output_file) && !@invalidated_files.include?(path) + trigger(:skipped, output_file) + return [output_file, nil] + end + elsif @glob + did_match = if defined?(::File::FNM_EXTGLOB) + File.fnmatch(@glob, resource.destination_path, ::File::FNM_EXTGLOB) + else + File.fnmatch(@glob, resource.destination_path) + end + + unless did_match + trigger(:skipped, output_file) + return [output_file, nil] + end + end + + deps = nil + if resource.binary? - export_file!(output_file, resource.file_descriptor[:full_path]) + export_file!(output_file, resource.file_descriptor[:full_path], true) else - export_file!(output_file, binary_encode(resource.render({}, {}))) + content = resource.render({}, {}) + + unless resource.dependencies.empty? + deps = ::Middleman::Dependencies::Dependency.new( + resource.source_file, + resource.dependencies + ) + end + + export_file!(output_file, binary_encode(content)) end rescue StandardError => e trigger(:error, output_file, "#{e}\n#{e.backtrace.join("\n")}") return false end - output_file + [output_file, deps] end end diff --git a/middleman-core/lib/middleman-core/dependencies.rb b/middleman-core/lib/middleman-core/dependencies.rb new file mode 100644 index 00000000..61881a90 --- /dev/null +++ b/middleman-core/lib/middleman-core/dependencies.rb @@ -0,0 +1,194 @@ +require 'middleman-core/contracts' +require 'set' +require 'pathname' +require 'yaml' + +module Middleman + module Dependencies + class Dependency + include Contracts + + Contract String + attr_reader :file + + Contract Maybe[SetOf[String]] + attr_accessor :depends_on + + Contract String, Maybe[SetOf[String]] => Any + def initialize(file, depends_on = nil) + @file = file + @depends_on = nil + @depends_on = Set.new(depends_on) unless depends_on.nil? + end + + Contract String + def to_s + "#<#{self.class} file=#{@file} depends_on=#{@depends_on}>" + end + end + + class Graph + include Contracts + + Contract HashOf[String, String] + attr_reader :hashes + + Contract HashOf[String, SetOf[String]] + attr_accessor :dependency_map + + def initialize(hashes = {}) + @hashes = hashes + @dependency_map = {} + end + + Contract Dependency => Any + def add_dependency(file) + @dependency_map[file.file] ||= Set.new + @dependency_map[file.file] << file.file + + return if file.depends_on.nil? + + file.depends_on.each do |dep| + @dependency_map[dep] ||= Set.new + @dependency_map[dep] << dep + @dependency_map[dep] << file.file + end + end + + Contract String => Bool + def exists?(file_path) + @dependency_map.key?(file_path) + end + + Contract SetOf[String] + def invalidated + invalidated_files = @dependency_map.keys.select do |file| + if !@hashes.key?(file) + # $stderr.puts "#{file} missing hash" + true + else + # $stderr.puts "#{file} invalid hash" + hashes[file] != ::Middleman::Dependencies.hashing_method(file) + end + end + + invalidated_files.reduce(Set.new) do |sum, file| + sum << file + sum | @dependency_map[file] + end + end + end + + include Contracts + + module_function + + Contract String => String + def hashing_method(file_name) + ::Digest::SHA1.file(file_name).hexdigest + end + + Contract ArrayOf[String] + def ruby_files_paths + Dir['**/*.rb', 'Gemfile.lock'] + end + + Contract IsA['::Middleman::Application'], String => String + def relativize(app, file) + Pathname(File.expand_path(file)).relative_path_from(app.root_path).to_s + end + + Contract IsA['::Middleman::Application'], String => String + def fullize(app, file) + File.expand_path(file, app.root_path) + end + + Contract IsA['::Middleman::Application'], Graph => String + def serialize(app, graph) + ruby_files = ruby_files_paths.reduce([]) do |sum, file| + sum << { + file: relativize(app, file), + hash: hashing_method(file) + } + end + + source_files = graph.dependency_map.reduce([]) do |sum, (file, depended_on_by)| + sum << { + file: relativize(app, file), + hash: hashing_method(file), + depended_on_by: depended_on_by.delete(file).to_a.sort.map { |p| relativize(app, p) } + } + end + + ::YAML.dump( + ruby_files: ruby_files.sort_by { |d| d[:file] }, + source_files: source_files.sort_by { |d| d[:file] } + ) + end + + DEFAULT_FILE_PATH = 'deps.yml'.freeze + + Contract IsA['::Middleman::Application'], Graph, Maybe[String] => Any + def serialize_and_save(app, graph, file_path = DEFAULT_FILE_PATH) + File.open(file_path, 'w') do |file| + file.write serialize(app, graph) + end + end + + Contract String => Graph + def deserialize(file_path) + ::YAML.load_file(file_path) + rescue StandardError, ::Psych::SyntaxError => error + warn "YAML Exception parsing dependency graph: #{error.message}" + end + + Contract ArrayOf[String] + def invalidated_ruby_files(known_files) + known_files.reject do |file| + file[:hash] == hashing_method(file[:file]) + end + end + + class InvalidDepsYAML < RuntimeError + end + + class InvalidatedRubyFiles < RuntimeError + attr_reader :invalidated + + def initialize(invalidated) + super() + + @invalidated = invalidated + end + end + + Contract IsA['::Middleman::Application'], Maybe[String] => Graph + def load_and_deserialize(app, file_path = DEFAULT_FILE_PATH) + return Graph.new unless File.exist?(file_path) + + data = deserialize(file_path) + + ruby_files = data[:ruby_files] + + unless (invalidated = invalidated_ruby_files(ruby_files)).empty? + raise InvalidatedRubyFiles, invalidated + end + + source_files = data[:source_files] + + hashes = source_files.each_with_object({}) do |row, sum| + sum[fullize(app, row[:file])] = row[:hash] + end + + graph = Graph.new(hashes) + + graph.dependency_map = source_files.each_with_object({}) do |row, sum| + sum[fullize(app, row[:file])] = Set.new((row[:depended_on_by] + [row[:file]]).map { |f| fullize(app, f) }) + end + + graph + rescue StandardError + raise InvalidDepsYAML + end + end +end diff --git a/middleman-core/lib/middleman-core/extension.rb b/middleman-core/lib/middleman-core/extension.rb index 1affa64e..43d98690 100644 --- a/middleman-core/lib/middleman-core/extension.rb +++ b/middleman-core/lib/middleman-core/extension.rb @@ -61,8 +61,6 @@ module Middleman # # There are also some less common hooks that can be listened to from within an extension's `initialize` method: # - # * `app.before_render {|body, path, locs, template_class| ... }` - Manipulate template sources before they are rendered. - # * `app.after_render {|content, path, locs, template_class| ... }` - Manipulate output text after a template has been rendered. # * `app.ready { ... }` - Run code once Middleman is ready to serve or build files (after `after_configuration`). # diff --git a/middleman-core/lib/middleman-core/extensions/asset_hash.rb b/middleman-core/lib/middleman-core/extensions/asset_hash.rb index 1becd32e..9b21db7d 100644 --- a/middleman-core/lib/middleman-core/extensions/asset_hash.rb +++ b/middleman-core/lib/middleman-core/extensions/asset_hash.rb @@ -1,7 +1,7 @@ require 'middleman-core/util' class Middleman::Extensions::AssetHash < ::Middleman::Extension - option :sources, %w[.css .htm .html .js .php .xhtml], 'List of extensions that are searched for hashable assets.' + option :sources, %w[.css .htm .html .js .json .php .xhtml], 'List of extensions that are searched for hashable assets.' option :exts, nil, 'List of extensions that get asset hashes appended to them.' option :ignore, [], 'Regexes of filenames to skip adding asset hashes to' option :rewrite_ignore, [], 'Regexes of filenames to skip processing for path rewrites' @@ -54,6 +54,7 @@ class Middleman::Extensions::AssetHash < ::Middleman::Extension r.add_filter ::Middleman::InlineURLRewriter.new(:asset_hash, app, r, + create_dependencies: true, url_extensions: @set_of_exts, ignore: options.ignore, proc: method(:rewrite_url)) diff --git a/middleman-core/lib/middleman-core/file_renderer.rb b/middleman-core/lib/middleman-core/file_renderer.rb index 46e8ae78..a4ace8a3 100644 --- a/middleman-core/lib/middleman-core/file_renderer.rb +++ b/middleman-core/lib/middleman-core/file_renderer.rb @@ -1,4 +1,5 @@ require 'tilt' +require 'set' require 'active_support/core_ext/string/output_safety' require 'active_support/core_ext/module/delegation' require 'middleman-core/contracts' @@ -15,11 +16,15 @@ module Middleman @_cache ||= ::Tilt::Cache.new end + Contract Maybe[SetOf[String]] + attr_reader :dependencies + def_delegator :"self.class", :cache def initialize(app, path) @app = app @path = path.to_s + @dependencies = nil end # Render an on-disk file. Used for everything, including layouts. @@ -60,28 +65,19 @@ module Middleman # Overwrite with frontmatter options options = options.deep_merge(options[:renderer_options]) if options[:renderer_options] - template_class = ::Middleman::Util.tilt_class(path) - - # Allow hooks to manipulate the template before render - body = @app.callbacks_for(:before_render).reduce(body) do |sum, callback| - callback.call(sum, path, locs, template_class) || sum - end - # Read compiled template from disk or cache template = ::Tilt.new(path, 1, options) { body } # template = cache.fetch(:compiled_template, extension, options, body) do # ::Tilt.new(path, 1, options) { body } # end - # Render using Tilt - # content = ::Middleman::Util.instrument 'render.tilt', path: path do - # template.render(context, locs, &block) - # end - content = template.render(context, locs, &block) + @dependencies = nil - # Allow hooks to manipulate the result after render - content = @app.callbacks_for(:after_render).reduce(content) do |sum, callback| - callback.call(sum, path, locs, template_class) || sum + # Render using Tilt + content = ::Middleman::Util.instrument 'render.tilt', path: path do + template.render(context, locs, &block).tap do + @dependencies = template.dependencies if template.respond_to?(:dependencies) + end end output = ::ActiveSupport::SafeBuffer.new '' diff --git a/middleman-core/lib/middleman-core/filter.rb b/middleman-core/lib/middleman-core/filter.rb index aa08d2e4..99b91316 100644 --- a/middleman-core/lib/middleman-core/filter.rb +++ b/middleman-core/lib/middleman-core/filter.rb @@ -45,9 +45,10 @@ module Middleman @callable = callable end - Contract String => String + Contract String => [String, Maybe[SetOf[String]]] def execute_filter(body) - @callable.call(body) + result = @callable.call(body) + result.is_a?(Array) ? result : [result, nil] end end end diff --git a/middleman-core/lib/middleman-core/inline_url_rewriter.rb b/middleman-core/lib/middleman-core/inline_url_filter.rb similarity index 55% rename from middleman-core/lib/middleman-core/inline_url_rewriter.rb rename to middleman-core/lib/middleman-core/inline_url_filter.rb index 92206752..2f36d3c5 100644 --- a/middleman-core/lib/middleman-core/inline_url_rewriter.rb +++ b/middleman-core/lib/middleman-core/inline_url_filter.rb @@ -14,13 +14,29 @@ module Middleman @resource = resource end - Contract String => String + Contract String, Pathname => Maybe[IsA['::Middleman::Sitemap::Resource']] + def target_resource(asset_path, dirpath) + uri = ::Middleman::Util.parse_uri(asset_path) + relative_path = !uri.path.start_with?('/') + + full_asset_path = if relative_path + dirpath.join(asset_path).to_s + else + asset_path + end + + @app.sitemap.by_destination_path(full_asset_path) || @app.sitemap.by_path(full_asset_path) + end + + Contract String => [String, Maybe[SetOf[String]]] def execute_filter(body) path = "/#{@resource.destination_path}" dirpath = ::Pathname.new(File.dirname(path)) - ::Middleman::Util.instrument 'inline_url_rewriter', path: path do - ::Middleman::Util.rewrite_paths(body, path, @options.fetch(:url_extensions), @app) do |asset_path| + ::Middleman::Util.instrument 'inline_url_filter', path: path do + deps = Set.new + + new_content = ::Middleman::Util.rewrite_paths(body, path, @options.fetch(:url_extensions), @app) do |asset_path| uri = ::Middleman::Util.parse_uri(asset_path) relative_path = uri.host.nil? @@ -36,10 +52,17 @@ module Middleman next if @options.fetch(:ignore).any? { |r| ::Middleman::Util.should_ignore?(r, full_asset_path) } result = @options.fetch(:proc).call(asset_path, dirpath, path) - asset_path = result if result + + if result + deps << target_resource(asset_path, dirpath).source_file if @options.fetch(:create_dependencies, false) + + asset_path = result + end asset_path end + + [new_content, deps] end end end diff --git a/middleman-core/lib/middleman-core/renderers/sass.rb b/middleman-core/lib/middleman-core/renderers/sass.rb index 43ea1f5f..2e5e3325 100644 --- a/middleman-core/lib/middleman-core/renderers/sass.rb +++ b/middleman-core/lib/middleman-core/renderers/sass.rb @@ -1,3 +1,4 @@ +require 'set' require 'sass' begin @@ -74,6 +75,11 @@ module Middleman end end + def dependencies + files = @engine.dependencies.map(&:filename) + files.empty? ? nil : Set.new(files) + end + # Change Sass path, for url functions, to the build folder if we're building # @return [Hash] def sass_options diff --git a/middleman-core/lib/middleman-core/sitemap/resource.rb b/middleman-core/lib/middleman-core/sitemap/resource.rb index 3423cf2c..35c2b705 100644 --- a/middleman-core/lib/middleman-core/sitemap/resource.rb +++ b/middleman-core/lib/middleman-core/sitemap/resource.rb @@ -4,7 +4,7 @@ require 'middleman-core/file_renderer' require 'middleman-core/template_renderer' require 'middleman-core/contracts' require 'set' -require 'middleman-core/inline_url_rewriter' +require 'middleman-core/inline_url_filter' module Middleman # Sitemap namespace @@ -37,6 +37,9 @@ module Middleman Contract Num attr_reader :priority + Contract Maybe[SetOf[String]] + attr_reader :dependencies + # Initialize resource with parent store and URL # @param [Middleman::Sitemap::Store] store # @param [String] path @@ -49,6 +52,7 @@ module Middleman @ignored = false @filters = ::Hamster::SortedSet.empty @priority = priority + @dependencies = Set.new source = Pathname(source) if source&.is_a?(String) @@ -169,6 +173,8 @@ module Middleman # @return [String] Contract Hash, Hash, Maybe[Proc] => String def render(options_hash = ::Middleman::EMPTY_HASH, locs = ::Middleman::EMPTY_HASH, &_block) + @dependencies = Set.new + body = render_without_filters(options_hash, locs) return body if @filters.empty? @@ -177,7 +183,9 @@ module Middleman if block_given? && !yield(filter) output elsif filter.is_a?(Filter) - filter.execute_filter(output) + result = filter.execute_filter(output) + @dependencies |= result[1] unless result[1].nil? + result[0] else output end @@ -208,7 +216,9 @@ module Middleman locs[:current_path] ||= destination_path renderer = ::Middleman::TemplateRenderer.new(@app, file_descriptor[:full_path].to_s) - renderer.render(locs, opts).to_str + renderer.render(locs, opts).to_str.tap do + @dependencies |= renderer.dependencies unless renderer.dependencies.nil? + end end # A path without the directory index - so foo/index.html becomes diff --git a/middleman-core/lib/middleman-core/template_context.rb b/middleman-core/lib/middleman-core/template_context.rb index 7dc9ec1e..de1d62eb 100644 --- a/middleman-core/lib/middleman-core/template_context.rb +++ b/middleman-core/lib/middleman-core/template_context.rb @@ -22,6 +22,9 @@ module Middleman # Required for Padrino's rendering attr_accessor :current_engine + Contract Maybe[SetOf[String]] + attr_reader :dependencies + # Shorthand references to global values on the app instance. def_delegators :@app, :config, :logger, :sitemap, :server?, :build?, :environment?, :environment, :data, :extensions, :root, :development?, :production? @@ -34,6 +37,8 @@ module Middleman @app = app @locs = locs @opts = options_hash + + @dependencies = Set.new end # Return the current buffer to the caller and clear the value internally. @@ -86,6 +91,8 @@ module Middleman restore_buffer(buf_was) end + @dependencies << layout_file[:full_path].to_s + # Render the layout, with the contents of the block inside. concat_safe_content render_file(layout_file, @locs, @opts) { content } ensure @@ -111,14 +118,18 @@ module Middleman source_path = sitemap.file_to_path(partial_file) r = sitemap.by_path(source_path) - if (r && !r.template?) || (Tilt[partial_file[:full_path]].nil? && partial_file[:full_path].exist?) - partial_file.read - else - opts = options_hash.dup - locs = opts.delete(:locals) + @dependencies << partial_file[:full_path].to_s - render_file(partial_file, locs, opts, &block) - end + result = if (r && !r.template?) || (Tilt[partial_file[:full_path]].nil? && partial_file[:full_path].exist?) + partial_file.read + else + opts = options_hash.dup + locs = opts.delete(:locals) + + render_file(partial_file, locs, opts, &block) + end + + result end # Locate a partial relative to the current path or the source dir, given a partial's path. @@ -207,6 +218,7 @@ module Middleman content_renderer = ::Middleman::FileRenderer.new(@app, path) content = content_renderer.render(locs, opts, context, &block) + @dependencies |= content_renderer.dependencies unless content_renderer.dependencies.nil? path = File.basename(path, File.extname(path)) rescue LocalJumpError diff --git a/middleman-core/lib/middleman-core/template_renderer.rb b/middleman-core/lib/middleman-core/template_renderer.rb index 7551022b..0829a360 100644 --- a/middleman-core/lib/middleman-core/template_renderer.rb +++ b/middleman-core/lib/middleman-core/template_renderer.rb @@ -1,4 +1,5 @@ require 'tilt' +require 'set' require 'active_support/core_ext/string/output_safety' require 'middleman-core/template_context' require 'middleman-core/file_renderer' @@ -98,9 +99,13 @@ module Middleman # Custom error class for handling class TemplateNotFound < RuntimeError; end + Contract Maybe[SetOf[String]] + attr_reader :dependencies + def initialize(app, path) @app = app @path = path + @dependencies = nil end # Render a template, with layout, given a path @@ -167,13 +172,19 @@ module Middleman layout_renderer = ::Middleman::FileRenderer.new(@app, layout_file[:relative_path].to_s) ::Middleman::Util.instrument 'builder.output.resource.render-layout', path: File.basename(layout_file[:relative_path].to_s) do - layout_renderer.render(locals, options, context) { content } + layout_renderer.render(locals, options, context) { content }.tap do + @dependencies << layout_file[:full_path].to_s + @dependencies |= layout_renderer.dependencies unless layout_renderer.dependencies.nil? + end end else content end end + @dependencies |= context.dependencies unless context.dependencies.nil? + @dependencies = @dependencies.empty? ? nil : @dependencies + # Return result content ensure @@ -189,12 +200,15 @@ module Middleman # handles cases like `style.css.sass.erb` content = nil + @dependencies = Set.new + while ::Middleman::Util.tilt_class(path) begin opts[:template_body] = content if content content_renderer = ::Middleman::FileRenderer.new(@app, path) content = content_renderer.render(locs, opts, context, &block) + @dependencies |= content_renderer.dependencies unless content_renderer.dependencies.nil? path = path.sub(/\.[^.]*\z/, '') rescue LocalJumpError diff --git a/middleman-core/lib/middleman-core/version.rb b/middleman-core/lib/middleman-core/version.rb index f03f26a8..dc0b4402 100644 --- a/middleman-core/lib/middleman-core/version.rb +++ b/middleman-core/lib/middleman-core/version.rb @@ -1,5 +1,5 @@ module Middleman # Current Version # @return [String] - VERSION = '4.3.0.rc.4'.freeze unless const_defined?(:VERSION) + VERSION = '5.0.0.rc.1'.freeze unless const_defined?(:VERSION) end diff --git a/vendor.yml b/vendor.yml deleted file mode 100644 index eec4cafe..00000000 --- a/vendor.yml +++ /dev/null @@ -1,3 +0,0 @@ -- fixtures -- templates -- vendored-middleman-deps