diff --git a/pkg/archive/changes.go b/pkg/archive/changes.go
index f2ac2a3561..c3cb4ebe0e 100644
--- a/pkg/archive/changes.go
+++ b/pkg/archive/changes.go
@@ -220,8 +220,8 @@ func (info *FileInfo) addChanges(oldInfo *FileInfo, changes *[]Change) {
 				oldStat.Gid() != newStat.Gid() ||
 				oldStat.Rdev() != newStat.Rdev() ||
 				// Don't look at size for dirs, its not a good measure of change
-				(oldStat.Size() != newStat.Size() && oldStat.Mode()&syscall.S_IFDIR != syscall.S_IFDIR) ||
-				!sameFsTimeSpec(oldStat.Mtim(), newStat.Mtim()) ||
+				(oldStat.Mode()&syscall.S_IFDIR != syscall.S_IFDIR &&
+					(!sameFsTimeSpec(oldStat.Mtim(), newStat.Mtim()) || (oldStat.Size() != newStat.Size()))) ||
 				bytes.Compare(oldChild.capability, newChild.capability) != 0 {
 				change := Change{
 					Path: newChild.path(),
diff --git a/pkg/archive/changes_test.go b/pkg/archive/changes_test.go
index 8f32d7b30e..53ec575b67 100644
--- a/pkg/archive/changes_test.go
+++ b/pkg/archive/changes_test.go
@@ -218,7 +218,6 @@ func TestChangesDirsMutated(t *testing.T) {
 	expectedChanges := []Change{
 		{"/dir1", ChangeDelete},
 		{"/dir2", ChangeModify},
-		{"/dir3", ChangeModify},
 		{"/dirnew", ChangeAdd},
 		{"/file1", ChangeDelete},
 		{"/file2", ChangeModify},