1
0
Fork 0
mirror of https://github.com/middleman/middleman.git synced 2022-11-09 12:20:27 -05:00

Use an actual graph for the dependency graph (#2233)

This commit is contained in:
Thomas Reynolds 2018-12-28 22:48:38 -08:00 committed by GitHub
parent 6a9b5c9cf2
commit 09fbd968ae
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
31 changed files with 366 additions and 145 deletions

2
.gitignore vendored
View file

@ -9,6 +9,8 @@ pkg
.sass-cache
.sassc
.tmp
graph.dot
graph.jpg
docs
.rbenv-*
.ruby-version

View file

@ -26,6 +26,7 @@ PATH
padrino-helpers (~> 0.13.0)
parallel
rack (>= 1.4.5, < 3)
rgl (~> 0.5.3)
sass (>= 3.4)
sassc (~> 2.0)
servolux
@ -113,6 +114,7 @@ GEM
jaro_winkler (1.5.1)
json (2.1.0)
kramdown (1.17.0)
lazy_priority_queue (0.1.1)
libv8 (6.7.288.46.1)
liquid (4.0.1)
listen (3.1.5)
@ -166,6 +168,9 @@ GEM
rb-inotify (0.10.0)
ffi (~> 1.0)
redcarpet (3.4.0)
rgl (0.5.3)
lazy_priority_queue (~> 0.1.0)
stream (~> 0.5.0)
rspec (3.8.0)
rspec-core (~> 3.8.0)
rspec-expectations (~> 3.8.0)
@ -215,6 +220,7 @@ GEM
slim (4.0.1)
temple (>= 0.7.6, < 0.9)
tilt (>= 2.0.6, < 2.1)
stream (0.5)
stylus (1.0.2)
execjs
stylus-source

View file

@ -44,6 +44,10 @@ module Middleman::Cli
type: :boolean,
default: false,
desc: 'Track file dependencies'
class_option :visualize_graph,
type: :boolean,
default: false,
desc: 'Generate a visual of the dependency graph'
class_option :only_changed,
type: :boolean,
default: false,
@ -95,7 +99,8 @@ module Middleman::Cli
parallel: options['parallel'],
only_changed: options['only_changed'],
missing_and_changed: missing_and_changed,
track_dependencies: should_track_dependencies)
track_dependencies: should_track_dependencies,
visualize_graph: options['visualize_graph'])
builder.thor = self
builder.on_build_event(&method(:on_event))
end

View file

@ -0,0 +1,174 @@
Feature: Incremental builds
Scenario: Changing a page should only rebuild that page
Given an empty app
When a file named "config.rb" with:
"""
"""
When a file named "source/standalone.html.erb" with:
"""
Initial
"""
When a file named "source/other.html.erb" with:
"""
Some other file
"""
Then build the app tracking dependencies
Then the output should contain "create build/standalone.html"
Then the following files should exist:
| build/standalone.html |
And the file "build/standalone.html" should contain "Initial"
When a file named "source/standalone.html.erb" with:
"""
Updated
"""
Then build app with only changed
Then there are "0" files which are " create "
Then there are "1" files which are " updated "
Then the output should contain "updated build/standalone.html"
Then the following files should exist:
| build/standalone.html |
And the file "build/standalone.html" should contain "Updated"
Scenario: Changing a layout should only rebuild pages which use that layout
Given an empty app
When a file named "config.rb" with:
"""
"""
When a file named "source/layout.erb" with:
"""
Initial
<%= yield %>
"""
When a file named "source/page1.html.erb" with:
"""
Page 1
"""
When a file named "source/page2.html.erb" with:
"""
Page 2
"""
When a file named "source/no-layout.html.erb" with:
"""
---
layout: false
---
Another page
"""
Then build the app tracking dependencies
Then the output should contain "create build/page1.html"
Then the output should contain "create build/page2.html"
Then the following files should exist:
| build/page1.html |
| build/page2.html |
And the file "build/page1.html" should contain "Initial"
And the file "build/page1.html" should contain "Page 1"
And the file "build/page2.html" should contain "Initial"
And the file "build/page2.html" should contain "Page 2"
When a file named "source/layout.erb" with:
"""
Updated
<%= yield %>
"""
Then build app with only changed
Then there are "0" files which are " create "
Then there are "2" files which are " updated "
Then the output should contain "updated build/page1.html"
Then the output should contain "updated build/page2.html"
Then the following files should exist:
| build/page1.html |
| build/page2.html |
And the file "build/page1.html" should contain "Updated"
And the file "build/page1.html" should contain "Page 1"
And the file "build/page2.html" should contain "Updated"
And the file "build/page2.html" should contain "Page 2"
Scenario: Changing a piece of data only rebuilds the pages which use it
Given an empty app
When a file named "config.rb" with:
"""
data.people.each do |p|
proxy "/person-#{p.slug}.html", '/person.html', ignore: true, locals: { person: p }
end
"""
When a file named "data/people.yml" with:
"""
-
slug: "one"
name: "Person One"
age: 5
-
slug: "two"
name: "Person Two"
age: 10
"""
When a file named "source/person.html.erb" with:
"""
<%= person.name %>
"""
Then build the app tracking dependencies
Then the output should contain "create build/person-one.html"
Then the output should contain "create build/person-two.html"
Then the following files should exist:
| build/person-one.html |
| build/person-two.html |
And the file "build/person-one.html" should contain "Person One"
And the file "build/person-two.html" should contain "Person Two"
When a file named "data/people.yml" with:
"""
-
slug: "one"
name: "Person One"
age: 15
-
slug: "two"
name: "Person Two"
age: 20
"""
Then build app with only changed
Then there are "0" files which are " create "
Then there are "0" files which are " updated "
Then the following files should exist:
| build/person-one.html |
| build/person-two.html |
When a file named "data/people.yml" with:
"""
-
slug: "one"
name: "Person One"
age: 5
-
slug: "two"
name: "Person Updated"
age: 10
"""
Then build app with only changed
Then there are "0" files which are " create "
Then there are "1" files which are " updated "
Then the output should contain "updated build/person-two.html"
Then the following files should exist:
| build/person-one.html |
| build/person-two.html |
And the file "build/person-two.html" should contain "Person Updated"
When a file named "data/people.yml" with:
"""
-
slug: "updated-slug"
name: "Person New Slug"
age: 5
-
slug: "two"
name: "Person Updated"
age: 10
"""
Then build app with only changed
Then there are "1" files which are " create "
Then there are "1" files which are " remove "
Then there are "0" files which are " updated "
Then the output should contain "create build/person-updated-slug.html"
Then the output should contain "remove build/person-one.html"
Then the following files should exist:
| build/person-updated-slug.html |
| build/person-two.html |
And the file "build/person-updated-slug.html" should contain "Person New Slug"

View file

@ -1,12 +1,16 @@
-
from: "/test1.html"
to: "/index.html"
guid: 0
-
from: "/test2.html"
to: "/index.html"
guid: 1
-
from: "/test3.html"
to: "/index.html"
guid: 2
-
from: "/test4.html"
to: "/index.html"
to: "/index.html"
guid: 3

View file

@ -0,0 +1,3 @@
<%= data.pages.first %>
<%#= data.pages.first(2) %>
<%#= data.pages.first(2).first %>

View file

@ -1,10 +1 @@
<h1>Welcome</h1>
<%= data.pages.first %>
<%= data.pages.first(2) %>
<%= data.pages.first(2).first %>
--
<%= data.pages.last %>
<%= data.pages.last(2) %>
<%= data.pages.last(2).last %>
<h1>Welcome</h1>

View file

@ -0,0 +1,3 @@
<%= data.pages.last %>
<%#= data.pages.last(2) %>
<%#= data.pages.last(2).last %>

View file

@ -1,5 +1,6 @@
<html>
<body>
<%#= data.pages[1].guid %>
<%= yield %>
</body>
</html>

View file

@ -0,0 +1,5 @@
---
layout: false
---
No layout

View file

@ -0,0 +1 @@
2

View file

@ -1,3 +0,0 @@
data.people.each do |p|
proxy "/#{p.slug}.html", '/person.html', ignore: true, locals: { person: p }
end

View file

@ -0,0 +1,3 @@
# data.people.each do |p|
# proxy "/#{p.slug}.html", '/person.html', ignore: true, locals: { person: p }
# end

View file

@ -40,6 +40,7 @@ module Middleman
@only_changed = options_hash.fetch(:only_changed, false)
@missing_and_changed = options_hash.fetch(:missing_and_changed, false)
@track_dependencies = options_hash.fetch(:track_dependencies, false)
@visualize_graph = @track_dependencies && options_hash.fetch(:visualize_graph, false)
@dry_run = options_hash.fetch(:dry_run)
@cleaning = !@dry_run && options_hash.fetch(:clean)
@ -94,7 +95,7 @@ module Middleman
prerender_css.tap do |resources|
if @track_dependencies
resources.each do |r|
@graph.add_edge(r[1]) unless r[1].nil?
@graph.add_edge_set(r[1]) unless r[1].nil?
end
end
end
@ -106,7 +107,7 @@ module Middleman
output_files.tap do |resources|
if @track_dependencies
resources.each do |r|
@graph.add_edge(r[1]) unless r[1].nil?
@graph.add_edge_set(r[1]) unless r[1].nil?
end
end
end
@ -115,9 +116,12 @@ module Middleman
::Middleman::Profiling.report('build')
unless @has_error
partial_update_with_no_changes = (@only_changed || @missing_and_changed) && !@invalidated_files.empty?
partial_update_with_no_changes = (@only_changed || @missing_and_changed) && @invalidated_files.empty?
::Middleman::Dependencies.serialize_and_save(@app, @graph) if @track_dependencies && !partial_update_with_no_changes
::Middleman::Dependencies.visualize_graph(@app, @graph) if @track_dependencies && @visualize_graph
::Middleman::Util.instrument 'builder.clean' do
clean! if @cleaning
end
@ -326,12 +330,11 @@ module Middleman
else
content = resource.render({}, {})
unless resource.vertices.empty?
vertices = ::Middleman::Dependencies::Edge.new(
::Middleman::Dependencies::FileVertex.from_resource(resource),
resource.vertices
)
end
self_vertex = ::Middleman::Dependencies::FileVertex.from_resource(resource)
vertices = ::Middleman::Dependencies::Edge.new(
self_vertex,
resource.vertices << self_vertex
)
export_file!(output_file, binary_encode(content))
end

View file

@ -47,12 +47,6 @@ module Middleman
end
end
def vertices_for_key(k)
@data_stores.reduce(::Hamster::Set.empty) do |sum, s|
sum | s.vertices_for_key(k)
end
end
def enhanced_data(k)
value = key(k)

View file

@ -54,10 +54,12 @@ module Middleman
end
def to_s
log_access(:__full_access__)
@data.to_s
end
def to_json
log_access(:__full_access__)
@data.to_a.to_json
end

View file

@ -23,10 +23,12 @@ module Middleman
end
def to_s
log_access(:__full_access__)
@data.to_s
end
def to_json
log_access(:__full_access__)
@data.to_h.to_json
end
end

View file

@ -29,11 +29,6 @@ module Middleman
Hamster::Set.empty
end
Contract Symbol => ImmutableSetOf[::Middleman::Dependencies::Vertex]
def vertices_for_key(_k)
Hamster::Set.empty
end
Contract Hash
def to_h
keys.each_with_object({}) do |k, sum|

View file

@ -27,11 +27,6 @@ module Middleman
Hamster::Set.new(@keys_to_vertex.values.flatten(1))
end
Contract Symbol => ImmutableSetOf[::Middleman::Dependencies::Vertex]
def vertices_for_key(k)
@keys_to_vertex[k] || ::Hamster::Set.empty
end
# Store static data hash
#
# @param [Symbol] name Name of the data, used for namespacing

View file

@ -26,12 +26,6 @@ module Middleman
@app = app
@local_data = {}
@paths_to_vertex = {}
end
Contract Symbol => ImmutableSetOf[::Middleman::Dependencies::Vertex]
def vertices_for_key(k)
@paths_to_vertex[k] || ::Hamster::Set.empty
end
Contract ArrayOf[IsA['Middleman::SourceFile']], ArrayOf[IsA['Middleman::SourceFile']] => Any
@ -69,16 +63,6 @@ module Middleman
data_branch = data_branch[dir.to_sym]
end
# For now, all files nested under a folder in `data/` will invalidate
# the whole folder.
if paths.empty?
@paths_to_vertex[basename.to_sym] ||= ::Hamster::Set.empty
@paths_to_vertex[basename.to_sym] <<= ::Middleman::Dependencies::FileVertex.from_source_file(@app, file)
else
@paths_to_vertex[paths.first.to_sym] ||= ::Hamster::Set.empty
@paths_to_vertex[paths.first.to_sym] <<= ::Middleman::Dependencies::FileVertex.from_source_file(@app, file)
end
data_branch[basename.to_sym] = data
end

View file

@ -56,11 +56,7 @@ module Middleman
end
def method_missing(method, *args, &block)
if @ctx.internal_data_store.key?(method)
@ctx.vertices |= @ctx.internal_data_store.vertices_for_key(method)
return @ctx.internal_data_store.proxied_data(method, self)
end
return @ctx.internal_data_store.proxied_data(method, self) if @ctx.internal_data_store.key?(method)
super
end

View file

@ -24,8 +24,16 @@ module Middleman
end
end
Contract IsA['::Middleman::Application'], Graph => Any
def visualize_graph(_app, graph)
require 'rgl/dot'
graph.graph.write_to_graphic_file('jpg', 'graph')
end
Contract IsA['::Middleman::Application'], Graph => String
def serialize(app, graph)
serialized = graph.serialize
ruby_files = Dir.glob(RUBY_FILES).reduce([]) do |sum, file|
sum << {
file: Pathname(File.expand_path(file)).relative_path_from(app.root_path).to_s,
@ -33,21 +41,10 @@ module Middleman
}
end
edges = graph.dependency_map.reduce([]) do |sum, (vertex, depended_on_by)|
sum << {
key: vertex.key,
depended_on_by: depended_on_by.delete(vertex).to_a.map(&:key).sort
}
end
vertices = graph.dependency_map.reduce([]) do |sum, (vertex, _depended_on_by)|
sum << vertex.serialize
end
::YAML.dump(
ruby_files: ruby_files.sort_by { |d| d[:file] },
edges: edges.sort_by { |d| d[:key] },
vertices: vertices.sort_by { |d| d[:key] }
{
ruby_files: ruby_files.sort_by { |d| d[:file] }
}.merge(serialized)
)
end
@ -105,17 +102,18 @@ module Middleman
end
end
graph = Graph.new(vertices)
graph = Graph.new
vertices.values.each { |v| graph.add_vertex(v) }
Contract ImmutableHashOf[Vertex, ImmutableSetOf[Vertex]]
edges = data[:edges]
graph.dependency_map = edges.reduce(::Hamster::Hash.empty) do |sum, row|
vertex = graph.vertices[row[:key]]
depended_on_by = row[:depended_on_by].map { |k| graph.vertices[k] }
sum.put(vertex, ::Hamster::Set.new(depended_on_by) << vertex)
data[:edges].each do |e|
graph.add_edge_by_key(e[:depends_on], e[:key])
end
graph.invalidate_changes!
# require 'rgl/dot'
# graph.graph.write_to_graphic_file('jpg', 'valid')
graph
rescue StandardError
raise InvalidDepsYAML

View file

@ -1,87 +1,121 @@
require 'set'
require 'rgl/adjacency'
require 'middleman-core/contracts'
require 'middleman-core/dependencies/vertices/vertex'
require 'middleman-core/dependencies/edge'
module Middleman
module Dependencies
class DirectedAdjacencyGraph < ::RGL::DirectedAdjacencyGraph
def add_vertex(v)
super(merged_vertex_or_new(v))
end
def add_edge(u, v)
super(merged_vertex_or_new(u), merged_vertex_or_new(v))
end
def remove_vertex(vertex)
each_adjacent(vertex) do |v|
remove_vertex(v) unless v == vertex
end
super(vertex)
end
def find_vertex_by_key(key)
vertices.find { |v| v.key == key }
end
protected
def merged_vertex_or_new(vertex)
found_vertex = find_vertex_by_key(vertex.key)
if found_vertex
found_vertex.merge!(vertex)
found_vertex
else
vertex
end
end
end
class Graph
include Contracts
Contract HashOf[Vertex::VERTEX_KEY, Vertex]
attr_reader :vertices
Contract DirectedAdjacencyGraph
attr_reader :graph
Contract ImmutableHashOf[Vertex, ImmutableSetOf[Vertex]]
attr_accessor :dependency_map
def initialize(vertices = {})
@vertices = vertices
@dependency_map = ::Hamster::Hash.empty
def initialize(_vertices = {})
@graph = DirectedAdjacencyGraph.new
end
Contract Vertex => Vertex
def merged_vertex_or_new(v)
if @vertices[v.key]
@vertices[v.key].merge!(v)
else
@vertices[v.key] = v
end
def invalidate_vertex!(vertex)
@graph.remove_vertex(vertex)
end
@vertices[v.key]
Contract Vertex => Any
def add_vertex(vertex)
@graph.add_vertex(vertex)
end
Contract Vertex, Vertex => Any
def add_edge(source, target)
if source == target
add_vertex(source)
else
@graph.add_edge(source, target)
end
end
Contract Symbol, Symbol => Any
def add_edge_by_key(source, target)
a = @graph.find_vertex_by_key(source)
b = @graph.find_vertex_by_key(target)
add_edge(a, b)
end
Contract Edge => Any
def add_edge(edge)
deduped_vertex = merged_vertex_or_new edge.vertex
# FIXME
# Depending on yourself (<< deduped_vertex)
# is only useful for files in source/ that can be depended on and also
# be their own route
@dependency_map = @dependency_map.put(deduped_vertex) do |v|
(v || ::Hamster::Set.empty) << deduped_vertex
end
def add_edge_set(edge)
return if edge.depends_on.nil?
edge.depends_on.each do |depended_on|
deduped_depended_on = merged_vertex_or_new depended_on
@dependency_map = @dependency_map.put(deduped_depended_on) do |v|
(v || ::Hamster::Set.empty) << deduped_depended_on << deduped_vertex
end
add_edge(depended_on, edge.vertex)
end
end
Contract String => Bool
def exists?(file_path)
@dependency_map.key?(file_path)
def serialize
edges = @graph.edges.map do |edge|
{
key: edge.target.key,
depends_on: edge.source.key
}
end
vertices = @graph.vertices.map(&:serialize)
{
edges: edges.sort_by { |d| [d[:key], d[:depends_on]] },
vertices: vertices.sort_by { |d| d[:key] }
}
end
Contract Any
def invalidate_changes!
@invalidated = @graph.vertices.reject(&:valid?)
@invalidated.each { |v| @graph.remove_vertex(v) }
end
Contract ImmutableSetOf[Vertex]
def invalidated
@_invalidated_cache ||= begin
invalidated_vertices = @dependency_map.keys.select do |vertex|
# Either "Missing from known vertices"
# Or invalided by the class
!@vertices.key?(vertex.key) || !vertex.valid?
end
invalidated_vertices.reduce(::Hamster::Set.empty) do |sum, vertex|
sum | invalidated_with_parents(vertex)
end
end
end
Contract Vertex => ImmutableSetOf[Vertex]
def invalidated_with_parents(vertex)
# TODO, recurse more?
@dependency_map[vertex] << vertex
::Hamster::Set.new(@invalidated)
end
Contract IsA['::Middleman::Sitemap::Resource'] => Bool
def invalidates_resource?(resource)
invalidated.any? { |d| d.invalidates_resource?(resource) }
@graph.vertices.none? { |d| d.matches_resource?(resource) }
end
end
end

View file

@ -38,7 +38,7 @@ module Middleman
end
Contract IsA['Middleman::Sitemap::Resource'] => Bool
def invalidates_resource?(resource)
def matches_resource?(resource)
resource.file_descriptor[:full_path].to_s == @full_path
end

View file

@ -42,7 +42,7 @@ module Middleman
end
Contract IsA['Middleman::Sitemap::Resource'] => Bool
def invalidates_resource?(_resource)
def matches_resource?(_resource)
false
end

View file

@ -6,21 +6,27 @@ end
Given /^a built app at "([^\"]*)"$/ do |path|
step %(a fixture app "#{path}")
step %(I run `middleman build --verbose --no-parallel`)
end
cwd = File.expand_path(aruba.current_directory)
step %(I set the environment variable "MM_ROOT" to "#{cwd}")
Then /^build the app tracking dependencies$/ do
step %(I run `middleman build --track-dependencies --no-parallel`)
step %(was successfully built)
end
step %(I run `middleman build --verbose`)
Then /^build app with only changed$/ do
step %(I run `middleman build --track-dependencies --only-changed --no-parallel`)
step %(was successfully built)
end
Given /^was successfully built$/ do
# step %(the output should contain "Project built successfully.")
step %(the output should contain "Project built successfully.")
step %(the exit status should be 0)
step %(a directory named "build" should exist)
end
Given /^was not successfully built$/ do
# step %(the output should not contain "Project built successfully.")
step %(the output should not contain "Project built successfully.")
step %(the exit status should not be 0)
step %(a directory named "build" should not exist)
end
@ -36,7 +42,7 @@ Given /^a built app at "([^\"]*)" with flags "([^\"]*)"$/ do |path, flags|
cwd = File.expand_path(aruba.current_directory)
step %(I set the environment variable "MM_ROOT" to "#{cwd}")
step %(I run `middleman build #{flags}`)
step %(I run `middleman build --no-parallel #{flags}`)
end
Given /^a successfully built app at "([^\"]*)" with flags "([^\"]*)"$/ do |path, flags|
@ -55,3 +61,8 @@ Given /^I run the interactive middleman server$/ do
step %(I set the environment variable "MM_ROOT" to "#{cwd}")
step %(I run `middleman server` interactively)
end
Then('there are {string} files which are {string}') do |num, str|
# $stderr.puts last_command_started.output
expect(last_command_started.output.scan(str).length).to be num.to_i
end

View file

@ -7,9 +7,17 @@ Given /^app "([^\"]*)" is using config "([^\"]*)"$/ do |_path, config_name|
end
Given /^an empty app$/ do
ENV['MM_ROOT'] = nil
# This step can be reentered from several places but we don't want
# to keep re-copying and re-cd-ing into ever-deeper directories
next if File.basename(expand_path('.')) == 'empty_app'
step %(a directory named "empty_app")
step %(I cd to "empty_app")
ENV['MM_ROOT'] = nil
cwd = File.expand_path(aruba.current_directory)
step %(I set the environment variable "MM_ROOT" to "#{cwd}")
end
Given /^a fixture app "([^\"]*)"$/ do |path|
@ -25,6 +33,9 @@ Given /^a fixture app "([^\"]*)"$/ do |path|
FileUtils.cp_r(target_path, expand_path('.'))
step %(I cd to "#{path}")
cwd = File.expand_path(aruba.current_directory)
step %(I set the environment variable "MM_ROOT" to "#{cwd}")
end
Then /^the file "([^\"]*)" has the contents$/ do |path, contents|

View file

@ -26,6 +26,7 @@ Gem::Specification.new do |s|
s.add_dependency('parallel')
s.add_dependency('servolux')
s.add_dependency('dotenv')
s.add_dependency('rgl', ['~> 0.5.3'])
# Helpers
s.add_dependency('activesupport', ['>= 4.2', '< 5.2'])