diff --git a/graph/graph.go b/graph/graph.go
index 7932fa2ed2..d34b6fb683 100644
--- a/graph/graph.go
+++ b/graph/graph.go
@@ -82,9 +82,6 @@ type Graph struct {
 	imageMutex       imageMutex // protect images in driver.
 	retained         *retainedLayers
 	tarSplitDisabled bool
-
-	parentRefs      map[string]int
-	parentRefsMutex sync.Mutex
 }
 
 // file names for ./graph/<ID>/
@@ -115,11 +112,10 @@ func NewGraph(root string, driver graphdriver.Driver) (*Graph, error) {
 	}
 
 	graph := &Graph{
-		root:       abspath,
-		idIndex:    truncindex.NewTruncIndex([]string{}),
-		driver:     driver,
-		retained:   &retainedLayers{layerHolders: make(map[string]map[string]struct{})},
-		parentRefs: make(map[string]int),
+		root:     abspath,
+		idIndex:  truncindex.NewTruncIndex([]string{}),
+		driver:   driver,
+		retained: &retainedLayers{layerHolders: make(map[string]map[string]struct{})},
 	}
 
 	// Windows does not currently support tarsplit functionality.
@@ -287,13 +283,6 @@ func (graph *Graph) Register(img *image.Image, layerData io.Reader) (err error)
 		return err
 	}
 	graph.idIndex.Add(img.ID)
-
-	graph.parentRefsMutex.Lock()
-	if img.Parent != "" {
-		graph.parentRefs[img.Parent]++
-	}
-	graph.parentRefsMutex.Unlock()
-
 	return nil
 }
 
@@ -356,10 +345,6 @@ func (graph *Graph) Delete(name string) error {
 	if err != nil {
 		return err
 	}
-	img, err := graph.Get(id)
-	if err != nil {
-		return err
-	}
 	tmp, err := graph.mktemp()
 	graph.idIndex.Delete(id)
 	if err == nil {
@@ -374,16 +359,6 @@ func (graph *Graph) Delete(name string) error {
 	}
 	// Remove rootfs data from the driver
 	graph.driver.Remove(id)
-
-	graph.parentRefsMutex.Lock()
-	if img.Parent != "" {
-		graph.parentRefs[img.Parent]--
-		if graph.parentRefs[img.Parent] == 0 {
-			delete(graph.parentRefs, img.Parent)
-		}
-	}
-	graph.parentRefsMutex.Unlock()
-
 	// Remove the trashed image directory
 	return os.RemoveAll(tmp)
 }
@@ -401,11 +376,9 @@ func (graph *Graph) Map() map[string]*image.Image {
 // The walking order is undetermined.
 func (graph *Graph) walkAll(handler func(*image.Image)) {
 	graph.idIndex.Iterate(func(id string) {
-		img, err := graph.Get(id)
-		if err != nil {
+		if img, err := graph.Get(id); err != nil {
 			return
-		}
-		if handler != nil {
+		} else if handler != nil {
 			handler(img)
 		}
 	})
@@ -433,10 +406,7 @@ func (graph *Graph) ByParent() map[string][]*image.Image {
 
 // HasChildren returns whether the given image has any child images.
 func (graph *Graph) HasChildren(img *image.Image) bool {
-	graph.parentRefsMutex.Lock()
-	refCount := graph.parentRefs[img.ID]
-	graph.parentRefsMutex.Unlock()
-	return refCount > 0
+	return len(graph.ByParent()[img.ID]) > 0
 }
 
 // Retain keeps the images and layers that are in the pulling chain so that
@@ -454,14 +424,13 @@ func (graph *Graph) Release(sessionID string, layerIDs ...string) {
 // A head is an image which is not the parent of another image in the graph.
 func (graph *Graph) Heads() map[string]*image.Image {
 	heads := make(map[string]*image.Image)
+	byParent := graph.ByParent()
 	graph.walkAll(func(image *image.Image) {
 		// If it's not in the byParent lookup table, then
 		// it's not a parent -> so it's a head!
-		graph.parentRefsMutex.Lock()
-		if _, exists := graph.parentRefs[image.ID]; !exists {
+		if _, exists := byParent[image.ID]; !exists {
 			heads[image.ID] = image
 		}
-		graph.parentRefsMutex.Unlock()
 	})
 	return heads
 }