mirror of
https://github.com/middleman/middleman.git
synced 2022-11-09 12:20:27 -05:00
Centralize file reading and avoid parsing non-template files for front matter (#2229)
* Centralize file reading Be able to tell the difference between static files and templates Check other places for static_file? Allow pipelines to still operate on js/css chmod after moving * Fix check on frontmatter parsing * Reuse var * Fix mime check * Handle non-file resources with static check * Fix .php files and minification
This commit is contained in:
Thomas Reynolds
GitHub
parent
2880e9860f
commit
4c2b89bf41
22 changed files with 128 additions and 44 deletions
|
|
@ -1,6 +1,7 @@
|
|||
master
|
||||
===
|
||||
|
||||
* Add `--dry-run` to run a build, but skip outputting to disk.
|
||||
* Incremental builds: `--track-dependencies` and `--only-changed` flags (#2220)
|
||||
* Remove Rack support in favor of `resource.filters << proc { |oldbody| newbody }`
|
||||
* `manipulate_resource_list_container!` as a faster, less functional approach.
|
||||
|
|
|
|||
|
|
@ -36,6 +36,10 @@ module Middleman::Cli
|
|||
type: :boolean,
|
||||
default: false,
|
||||
desc: 'Generate profiling report for the build'
|
||||
class_option :dry_run,
|
||||
type: :boolean,
|
||||
default: false,
|
||||
desc: 'Skip writing output files'
|
||||
class_option :track_dependencies,
|
||||
type: :boolean,
|
||||
default: false,
|
||||
|
|
@ -86,6 +90,7 @@ module Middleman::Cli
|
|||
|
||||
builder = Middleman::Builder.new(@app,
|
||||
glob: options['glob'],
|
||||
dry_run: options['dry_run'],
|
||||
clean: options['clean'],
|
||||
parallel: options['parallel'],
|
||||
only_changed: options['only_changed'],
|
||||
|
|
|
|||
|
|
@ -55,7 +55,7 @@ Feature: Builder
|
|||
Scenario: Builded text file(ex: html, css, xml, txt)'s permission is 0644
|
||||
Given a successfully built app at "large-build-app"
|
||||
When I cd to "build"
|
||||
Then the mode of filesystem object "index.html" should match "0644"
|
||||
And the mode of filesystem object "stylesheets/static.css" should match "0644"
|
||||
And the mode of filesystem object "feed.xml" should match "0644"
|
||||
And the mode of filesystem object ".htaccess" should match "0644"
|
||||
Then the file named "index.html" should have permissions "0644"
|
||||
And the file named "stylesheets/static.css" should have permissions "0644"
|
||||
And the file named "feed.xml" should have permissions "0644"
|
||||
And the file named ".htaccess" should have permissions "0644"
|
||||
|
|
|
|||
|
|
@ -322,13 +322,13 @@ module Middleman
|
|||
config_rb = File.join(root, 'config.rb')
|
||||
if File.exist? config_rb
|
||||
logger.debug '== Reading: Local config: config.rb'
|
||||
config_context.instance_eval File.read(config_rb), config_rb, 1
|
||||
config_context.instance_eval ::Middleman::Util.read_file(config_rb), config_rb, 1
|
||||
else
|
||||
# Check for and evaluate local configuration in `middleman.rb`
|
||||
middleman_rb = File.join(root, 'middleman.rb')
|
||||
if File.exist? middleman_rb
|
||||
logger.debug '== Reading: Local middleman: middleman.rb'
|
||||
config_context.instance_eval File.read(middleman_rb), middleman_rb, 1
|
||||
config_context.instance_eval ::Middleman::Util.read_file(middleman_rb), middleman_rb, 1
|
||||
end
|
||||
end
|
||||
|
||||
|
|
@ -336,7 +336,7 @@ module Middleman
|
|||
return unless File.exist? env_config
|
||||
|
||||
logger.debug "== Reading: #{config[:environment]} config"
|
||||
config_context.instance_eval File.read(env_config), env_config, 1
|
||||
config_context.instance_eval ::Middleman::Util.read_file(env_config), env_config, 1
|
||||
end
|
||||
|
||||
# Clean up missing Tilt exts
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ module Middleman
|
|||
def_delegator :@app, :logger
|
||||
|
||||
# Sort order, images, fonts, js/css and finally everything else.
|
||||
SORT_ORDER = %w[.png .jpeg .jpg .gif .bmp .svg .svgz .webp .ico .woff .woff2 .otf .ttf .eot .js .css].freeze
|
||||
SORT_ORDER = %w[.png .jpeg .jpg .gif .bmp .svg .svgz .webp .ico .woff .woff2 .otf .ttf .eot .js .mjs .css].freeze
|
||||
|
||||
# Create a new Builder instance.
|
||||
# @param [Middleman::Application] app The app to build.
|
||||
|
|
@ -40,7 +40,8 @@ module Middleman
|
|||
@only_changed = options_hash.fetch(:only_changed, false)
|
||||
@missing_and_changed = options_hash.fetch(:missing_and_changed, false)
|
||||
@track_dependencies = options_hash.fetch(:track_dependencies, false)
|
||||
@cleaning = options_hash.fetch(:clean)
|
||||
@dry_run = options_hash.fetch(:dry_run)
|
||||
@cleaning = !@dry_run && options_hash.fetch(:clean)
|
||||
|
||||
@callbacks = ::Middleman::CallbackManager.new
|
||||
@callbacks.install_methods!(self, [:on_build_event])
|
||||
|
|
@ -253,7 +254,6 @@ module Middleman
|
|||
])
|
||||
file.binmode
|
||||
file.write(contents)
|
||||
File.chmod(0o644, file)
|
||||
file.close
|
||||
file
|
||||
end
|
||||
|
|
@ -264,6 +264,8 @@ module Middleman
|
|||
# @return [void]
|
||||
Contract Pathname, Or[String, Pathname], Maybe[Bool] => Any
|
||||
def export_file!(output_file, source, binary = false)
|
||||
return if @dry_run
|
||||
|
||||
::Middleman::Util.instrument 'write_file', output_file: output_file do
|
||||
source = write_tempfile(output_file, source.to_s) if source.is_a? String
|
||||
|
||||
|
|
@ -278,6 +280,7 @@ module Middleman
|
|||
if %i[created updated].include?(mode)
|
||||
::FileUtils.mkdir_p(output_file.dirname)
|
||||
method.call(source_path, output_file.to_s)
|
||||
::File.chmod(0o644, output_file.to_s)
|
||||
end
|
||||
|
||||
source.unlink if source.is_a? Tempfile
|
||||
|
|
@ -318,7 +321,7 @@ module Middleman
|
|||
|
||||
vertices = nil
|
||||
|
||||
if resource.binary?
|
||||
if resource.binary? || resource.static_file?
|
||||
export_file!(output_file, resource.file_descriptor[:full_path], true)
|
||||
else
|
||||
content = resource.render({}, {})
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ module Middleman
|
|||
|
||||
return unless File.exist? other_config
|
||||
|
||||
instance_eval File.read(other_config), other_config, 1
|
||||
instance_eval ::Middleman::Util.read_file(other_config), other_config, 1
|
||||
end
|
||||
|
||||
def set(key, default = nil, &block)
|
||||
|
|
|
|||
|
|
@ -86,10 +86,14 @@ module Middleman::CoreExtensions
|
|||
file_path = file[:full_path].to_s
|
||||
|
||||
@cache[file_path] ||= begin
|
||||
::Middleman::Util::Data.parse(
|
||||
file,
|
||||
app.config[:frontmatter_delims]
|
||||
)
|
||||
if ::Middleman::Util.contains_frontmatter?(file_path, app.config[:frontmatter_delims])
|
||||
::Middleman::Util::Data.parse(
|
||||
file,
|
||||
app.config[:frontmatter_delims]
|
||||
)
|
||||
else
|
||||
[{}, nil]
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
require 'hamster'
|
||||
require 'set'
|
||||
require 'pathname'
|
||||
require 'digest/sha1'
|
||||
require 'yaml'
|
||||
require 'middleman-core/contracts'
|
||||
require 'middleman-core/dependencies/graph'
|
||||
|
|
@ -30,7 +29,7 @@ module Middleman
|
|||
ruby_files = Dir.glob(RUBY_FILES).reduce([]) do |sum, file|
|
||||
sum << {
|
||||
file: Pathname(File.expand_path(file)).relative_path_from(app.root_path).to_s,
|
||||
hash: ::Digest::SHA1.file(file).hexdigest
|
||||
hash: ::Middleman::Util.hash_file(file)
|
||||
}
|
||||
end
|
||||
|
||||
|
|
@ -62,7 +61,7 @@ module Middleman
|
|||
Contract ArrayOf[String]
|
||||
def invalidated_ruby_files(known_files)
|
||||
known_files.reject do |file|
|
||||
file[:hash] == ::Digest::SHA1.file(file[:file]).hexdigest
|
||||
file[:hash] == ::Middleman::Util.hash_file(file[:file])
|
||||
end
|
||||
end
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
require 'hamster'
|
||||
require 'digest/sha1'
|
||||
require 'middleman-core/contracts'
|
||||
require 'middleman-core/dependencies/vertices/vertex'
|
||||
require 'middleman-core/core_extensions/data/controller'
|
||||
|
|
@ -47,7 +46,7 @@ module Middleman
|
|||
def current_hash
|
||||
@current_hash ||= begin
|
||||
data = lookup_path(key_to_path)
|
||||
::Digest::SHA1.hexdigest(data.to_s)
|
||||
::Middleman::Util.hash_string(data.to_s)
|
||||
end
|
||||
end
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
require 'hamster'
|
||||
require 'digest/sha1'
|
||||
require 'middleman-core/contracts'
|
||||
require 'middleman-core/dependencies/vertices/vertex'
|
||||
|
||||
|
|
@ -56,7 +55,7 @@ module Middleman
|
|||
def current_hash
|
||||
return nil if @data.nil?
|
||||
|
||||
@current_hash ||= ::Digest::SHA1.hexdigest(@data.to_s)
|
||||
@current_hash ||= ::Middleman::Util.hash_string(@data.to_s)
|
||||
end
|
||||
|
||||
Contract Maybe[String]
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
require 'pathname'
|
||||
require 'digest/sha1'
|
||||
require 'middleman-core/contracts'
|
||||
require 'middleman-core/dependencies/vertices/vertex'
|
||||
|
||||
|
|
@ -59,7 +58,7 @@ module Middleman
|
|||
|
||||
Contract String
|
||||
def hash_file
|
||||
::Digest::SHA1.file(@full_path).hexdigest
|
||||
::Middleman::Util.hash_file(@full_path)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
require 'middleman-core/util'
|
||||
|
||||
class Middleman::Extensions::AssetHash < ::Middleman::Extension
|
||||
option :sources, %w[.css .htm .html .js .json .php .xhtml], 'List of extensions that are searched for hashable assets.'
|
||||
option :sources, %w[.css .htm .html .js .mjs .json .php .xhtml], 'List of extensions that are searched for hashable assets.'
|
||||
option :exts, nil, 'List of extensions that get asset hashes appended to them.'
|
||||
option :ignore, [], 'Regexes of filenames to skip adding asset hashes to'
|
||||
option :rewrite_ignore, [], 'Regexes of filenames to skip processing for path rewrites'
|
||||
|
|
@ -10,8 +10,6 @@ class Middleman::Extensions::AssetHash < ::Middleman::Extension
|
|||
def initialize(app, options_hash = ::Middleman::EMPTY_HASH, &block)
|
||||
super
|
||||
|
||||
require 'digest/sha1'
|
||||
|
||||
# Allow specifying regexes to ignore, plus always ignore apple touch icons
|
||||
@ignore = Array(options.ignore) + [/^apple-touch-icon/]
|
||||
|
||||
|
|
@ -66,7 +64,7 @@ class Middleman::Extensions::AssetHash < ::Middleman::Extension
|
|||
sorted_resources = resource_list.by_extensions(@set_of_exts).sort_by do |a|
|
||||
if %w[.svg .svgz].include? a.ext
|
||||
0
|
||||
elsif %w[.js .css].include? a.ext
|
||||
elsif %w[.js .mjs .css].include? a.ext
|
||||
1
|
||||
else
|
||||
-1
|
||||
|
|
@ -83,12 +81,12 @@ class Middleman::Extensions::AssetHash < ::Middleman::Extension
|
|||
return if ignored_resource?(resource)
|
||||
return if resource.ignored?
|
||||
|
||||
digest = if resource.binary?
|
||||
::Digest::SHA1.file(resource.source_file).hexdigest[0..7]
|
||||
digest = if resource.binary? || resource.static_file?
|
||||
::Middleman::Util.hash_file(resource.source_file)[0..7]
|
||||
else
|
||||
# Render without asset hash
|
||||
body = resource.render({}, {}) { |f| !f.respond_to?(:filter_name) || f.filter_name != :asset_hash }
|
||||
::Digest::SHA1.hexdigest(body)[0..7]
|
||||
::Middleman::Util.hash_string(body)[0..7]
|
||||
end
|
||||
|
||||
resource_list.update!(resource, :destination_path) do
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
class Middleman::Extensions::AssetHost < ::Middleman::Extension
|
||||
option :host, nil, 'The asset host to use or a Proc to determine asset host', required: true
|
||||
option :exts, nil, 'List of extensions that get cache busters strings appended to them.'
|
||||
option :sources, %w[.css .htm .html .js .php .xhtml], 'List of extensions that are searched for bustable assets.'
|
||||
option :sources, %w[.css .htm .html .js .mjs .php .xhtml], 'List of extensions that are searched for bustable assets.'
|
||||
option :ignore, [], 'Regexes of filenames to skip adding query strings to'
|
||||
option :rewrite_ignore, [], 'Regexes of filenames to skip processing for host rewrites'
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
# The Cache Buster extension
|
||||
class Middleman::Extensions::CacheBuster < ::Middleman::Extension
|
||||
option :exts, nil, 'List of extensions that get cache busters strings appended to them.'
|
||||
option :sources, %w[.css .htm .html .js .php .xhtml], 'List of extensions that are searched for bustable assets.'
|
||||
option :sources, %w[.css .htm .html .js .mjs .php .xhtml], 'List of extensions that are searched for bustable assets.'
|
||||
option :ignore, [], 'Regexes of filenames to skip adding query strings to'
|
||||
option :rewrite_ignore, [], 'Regexes of filenames to skip processing for path rewrites'
|
||||
|
||||
|
|
|
|||
|
|
@ -7,10 +7,10 @@
|
|||
# to serve your Gzipped files whenever the normal (non-.gz) filename is requested.
|
||||
#
|
||||
# Pass the :exts options to customize which file extensions get zipped (defaults
|
||||
# to .css, .htm, .html, .js, and .xhtml
|
||||
# to .css, .htm, .html, .js, .mjs, and .xhtml
|
||||
#
|
||||
class Middleman::Extensions::Gzip < ::Middleman::Extension
|
||||
option :exts, %w[.css .htm .html .js .svg .xhtml], 'File extensions to Gzip when building.'
|
||||
option :exts, %w[.css .htm .html .js .mjs .svg .xhtml], 'File extensions to Gzip when building.'
|
||||
option :ignore, [], 'Patterns to avoid gzipping'
|
||||
option :overwrite, false, 'Overwrite original files instead of adding .gz extension.'
|
||||
|
||||
|
|
|
|||
|
|
@ -102,7 +102,7 @@ module Middleman
|
|||
return result unless result.nil?
|
||||
end
|
||||
|
||||
file ? file.read : ::File.read(@path)
|
||||
file ? file.read : ::Middleman::Util.read_file(@path)
|
||||
end
|
||||
|
||||
protected
|
||||
|
|
|
|||
|
|
@ -98,7 +98,7 @@ module Middleman
|
|||
return not_found(res, full_request_path) unless resource && !resource.ignored?
|
||||
|
||||
# If this path is a binary file, send it immediately
|
||||
return send_file(resource, env) if resource.binary?
|
||||
return send_file(resource, env) if resource.binary? || resource.static_file?
|
||||
|
||||
res['Content-Type'] = resource.content_type || 'text/plain'
|
||||
|
||||
|
|
@ -141,7 +141,10 @@ module Middleman
|
|||
response[1]['Content-Encoding'] = 'gzip' if %w[.svgz .gz].include?(resource.ext)
|
||||
# Do not set Content-Type if status is 1xx, 204, 205 or 304, otherwise
|
||||
# Rack will throw an error (500)
|
||||
response[1]['Content-Type'] = resource.content_type || 'application/octet-stream' if !(100..199).cover?(status) && ![204, 205, 304].include?(status)
|
||||
if !(100..199).cover?(status) && ![204, 205, 304].include?(status)
|
||||
response[1]['Content-Type'] = resource.content_type || (resource.binary? ? 'application/octet-stream' : 'text/plain')
|
||||
end
|
||||
|
||||
halt response
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -91,6 +91,13 @@ module Middleman
|
|||
!::Middleman::Util.tilt_class(file_descriptor[:full_path].to_s).nil?
|
||||
end
|
||||
|
||||
Contract Bool
|
||||
def static_file?
|
||||
return false if file_descriptor.nil?
|
||||
|
||||
::Middleman::Util.static_file?(file_descriptor[:full_path].to_s, app.config[:frontmatter_delims])
|
||||
end
|
||||
|
||||
# Backwards compatible method for turning descriptor into a string.
|
||||
# @return [String]
|
||||
Contract Maybe[String]
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ module Middleman
|
|||
SourceFile = Struct.new(:relative_path, :full_path, :directory, :types, :version) do
|
||||
def read
|
||||
::Middleman::Sources.file_cache[full_path] ||= {}
|
||||
::Middleman::Sources.file_cache[full_path][version] ||= ::File.read(full_path)
|
||||
::Middleman::Sources.file_cache[full_path][version] ||= ::Middleman::Util.read_file(full_path.to_s)
|
||||
end
|
||||
|
||||
def normalized_relative_path
|
||||
|
|
|
|||
|
|
@ -5,6 +5,15 @@ require 'set'
|
|||
|
||||
require 'middleman-core/contracts'
|
||||
|
||||
KNOWN_NON_STATIC_FILE_EXTENSIONS = Set.new %w[
|
||||
css
|
||||
js
|
||||
mjs
|
||||
html
|
||||
htm
|
||||
php
|
||||
]
|
||||
|
||||
KNOWN_BINARY_FILE_EXTENSIONS = Set.new %w[
|
||||
3dm
|
||||
3ds
|
||||
|
|
@ -330,5 +339,44 @@ module Middleman
|
|||
|
||||
false
|
||||
end
|
||||
|
||||
Contract String, Maybe[HashOf[Symbol, Any]] => Bool
|
||||
def static_file?(path, frontmatter_delims)
|
||||
path = Pathname(path)
|
||||
ext = path.extname
|
||||
without_dot = ext.sub('.', '')
|
||||
|
||||
if KNOWN_NON_STATIC_FILE_EXTENSIONS.include?(without_dot) || contains_frontmatter?(path, frontmatter_delims)
|
||||
false
|
||||
else
|
||||
!::Tilt.registered?(without_dot)
|
||||
end
|
||||
end
|
||||
|
||||
Contract String, Maybe[HashOf[Symbol, Any]] => Bool
|
||||
def contains_frontmatter?(path, frontmatter_delims)
|
||||
file = ::File.open(path)
|
||||
first_line = file.gets
|
||||
|
||||
first_line = file.gets if first_line =~ /\A(?:[^\r\n]*coding:[^\r\n]*\r?\n)/
|
||||
|
||||
file.close
|
||||
|
||||
possible_openers = possible_delim_openers(frontmatter_delims)
|
||||
!first_line.nil? && !first_line.match(possible_openers).nil?
|
||||
rescue EOFError, IOError, ::Errno::ENOENT
|
||||
false
|
||||
end
|
||||
|
||||
Contract Maybe[HashOf[Symbol, Any]] => Regexp
|
||||
def possible_delim_openers(frontmatter_delims)
|
||||
all_possible = frontmatter_delims
|
||||
.values
|
||||
.flatten(1)
|
||||
.map(&:first)
|
||||
.uniq
|
||||
|
||||
/\A#{::Regexp.union(all_possible)}/
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -45,13 +45,14 @@ module Middleman
|
|||
# @param [String] path
|
||||
# @return [Array<Hash, String>]
|
||||
Contract IsA['Middleman::SourceFile'], Maybe[Symbol] => [Hash, Maybe[String]]
|
||||
def parse(file, frontmatter_delims, known_type = nil)
|
||||
full_path = file[:full_path]
|
||||
return [{}, nil] if ::Middleman::Util.binary?(full_path) || file[:types].include?(:binary)
|
||||
def parse(source_file, frontmatter_delims, known_type = nil)
|
||||
full_path = source_file[:full_path]
|
||||
|
||||
return [{}, nil] if ::Middleman::Util.binary?(full_path) || source_file[:types].include?(:binary)
|
||||
|
||||
# Avoid weird race condition when a file is renamed
|
||||
begin
|
||||
content = file.read
|
||||
content = source_file.read
|
||||
rescue EOFError, IOError, ::Errno::ENOENT
|
||||
return [{}, nil]
|
||||
end
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
require 'digest/sha1'
|
||||
require 'set'
|
||||
|
||||
module Middleman
|
||||
|
|
@ -130,5 +131,22 @@ module Middleman
|
|||
end
|
||||
end.map(&:file_descriptor)
|
||||
end
|
||||
|
||||
Contract String, Maybe[Num] => String
|
||||
def read_file(path, bytes = nil)
|
||||
# puts "Read: #{path}"
|
||||
File.read(path, bytes)
|
||||
end
|
||||
|
||||
Contract String => String
|
||||
def hash_file(path)
|
||||
# puts "Read (hash): #{path}"
|
||||
::Digest::SHA1.file(path).hexdigest
|
||||
end
|
||||
|
||||
Contract String => String
|
||||
def hash_string(data)
|
||||
::Digest::SHA1.hexdigest(data)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue