devmapper: Store metadata in one file per device

This allows multiple instances of the backend in different containers
to access devices (although generally only one can modify/create them).

Any old metadata is converted on the first run.

Docker-DCO-1.1-Signed-off-by: Alexander Larsson <alexl@redhat.com> (github: alexlarsson)

daemon/graphdriver/devmapper/deviceset.go

@@ -106,7 +106,19 @@ func (devices *DeviceSet) loopbackDir() string {
 	return path.Join(devices.root, "devicemapper")
 }
 
-func (devices *DeviceSet) jsonFile() string {
+func (devices *DeviceSet) metadataDir() string {
+	return path.Join(devices.root, "metadata")
+}
+
+func (devices *DeviceSet) metadataFile(info *DevInfo) string {
+	file := info.Hash
+	if file == "" {
+		file = "base"
+	}
+	return path.Join(devices.metadataDir(), file)
+}
+
+func (devices *DeviceSet) oldMetadataFile() string {
 	return path.Join(devices.loopbackDir(), "json")
 }
 
@@ -161,14 +173,19 @@ func (devices *DeviceSet) allocateTransactionId() uint64 {
 	return devices.NewTransactionId
 }
 
-func (devices *DeviceSet) saveMetadata() error {
+func (devices *DeviceSet) removeMetadata(info *DevInfo) error {
-	devices.devicesLock.Lock()
+	if err := osRemoveAll(devices.metadataFile(info)); err != nil {
-	jsonData, err := json.Marshal(devices.MetaData)
+		return fmt.Errorf("Error removing metadata file %s: %s", devices.metadataFile(info), err)
-	devices.devicesLock.Unlock()
+	}
+	return nil
+}
+
+func (devices *DeviceSet) saveMetadata(info *DevInfo) error {
+	jsonData, err := json.Marshal(info)
 	if err != nil {
 		return fmt.Errorf("Error encoding metadata to json: %s", err)
 	}
-	tmpFile, err := ioutil.TempFile(filepath.Dir(devices.jsonFile()), ".json")
+	tmpFile, err := ioutil.TempFile(devices.metadataDir(), ".tmp")
 	if err != nil {
 		return fmt.Errorf("Error creating metadata file: %s", err)
 	}
@@ -186,7 +203,7 @@ func (devices *DeviceSet) saveMetadata() error {
 	if err := tmpFile.Close(); err != nil {
 		return fmt.Errorf("Error closing metadata file %s: %s", tmpFile.Name(), err)
 	}
-	if err := osRename(tmpFile.Name(), devices.jsonFile()); err != nil {
+	if err := osRename(tmpFile.Name(), devices.metadataFile(info)); err != nil {
 		return fmt.Errorf("Error committing metadata file %s: %s", tmpFile.Name(), err)
 	}
 
@@ -204,7 +221,12 @@ func (devices *DeviceSet) lookupDevice(hash string) (*DevInfo, error) {
 	defer devices.devicesLock.Unlock()
 	info := devices.Devices[hash]
 	if info == nil {
-		return nil, fmt.Errorf("Unknown device %s", hash)
+		info = devices.loadMetadata(hash)
+		if info == nil {
+			return nil, fmt.Errorf("Unknown device %s", hash)
+		}
+
+		devices.Devices[hash] = info
 	}
 	return info, nil
 }
@@ -224,7 +246,7 @@ func (devices *DeviceSet) registerDevice(id int, hash string, size uint64) (*Dev
 	devices.Devices[hash] = info
 	devices.devicesLock.Unlock()
 
-	if err := devices.saveMetadata(); err != nil {
+	if err := devices.saveMetadata(info); err != nil {
 		// Try to remove unused device
 		devices.devicesLock.Lock()
 		delete(devices.Devices, hash)
@@ -259,9 +281,7 @@ func (devices *DeviceSet) createFilesystem(info *DevInfo) error {
 	return nil
 }
 
-func (devices *DeviceSet) loadMetaData() error {
+func (devices *DeviceSet) initMetaData() error {
-	utils.Debugf("loadMetadata()")
-	defer utils.Debugf("loadMetadata END")
 	_, _, _, params, err := getStatus(devices.getPoolName())
 	if err != nil {
 		utils.Debugf("\n--->Err: %s\n", err)
@@ -274,35 +294,64 @@ func (devices *DeviceSet) loadMetaData() error {
 	}
 	devices.NewTransactionId = devices.TransactionId
 
-	jsonData, err := ioutil.ReadFile(devices.jsonFile())
+	// Migrate old metadatafile
+
+	jsonData, err := ioutil.ReadFile(devices.oldMetadataFile())
 	if err != nil && !osIsNotExist(err) {
 		utils.Debugf("\n--->Err: %s\n", err)
 		return err
 	}
 
-	devices.MetaData.Devices = make(map[string]*DevInfo)
 	if jsonData != nil {
-		if err := json.Unmarshal(jsonData, &devices.MetaData); err != nil {
+		m := MetaData{Devices: make(map[string]*DevInfo)}
+
+		if err := json.Unmarshal(jsonData, &m); err != nil {
 			utils.Debugf("\n--->Err: %s\n", err)
 			return err
 		}
-	}
 
-	for hash, d := range devices.Devices {
+		for hash, info := range m.Devices {
-		d.Hash = hash
+			info.Hash = hash
-		d.devices = devices
 
-		// If the transaction id is larger than the actual one we lost the device due to some crash
+			// If the transaction id is larger than the actual one we lost the device due to some crash
-		if d.TransactionId > devices.TransactionId {
+			if info.TransactionId <= devices.TransactionId {
-			utils.Debugf("Removing lost device %s with id %d", hash, d.TransactionId)
+				devices.saveMetadata(info)
-			delete(devices.Devices, hash)
+			}
 		}
+		if err := osRename(devices.oldMetadataFile(), devices.oldMetadataFile()+".migrated"); err != nil {
+			return err
+		}
+
 	}
+
 	return nil
 }
 
+func (devices *DeviceSet) loadMetadata(hash string) *DevInfo {
+	info := &DevInfo{Hash: hash, devices: devices}
+
+	jsonData, err := ioutil.ReadFile(devices.metadataFile(info))
+	if err != nil {
+		return nil
+	}
+
+	if err := json.Unmarshal(jsonData, &info); err != nil {
+		return nil
+	}
+
+	fmt.Printf("Loaded metadata %v\n", info)
+
+	// If the transaction id is larger than the actual one we lost the device due to some crash
+	if info.TransactionId > devices.TransactionId {
+		return nil
+	}
+
+	return info
+}
+
 func (devices *DeviceSet) setupBaseImage() error {
 	oldInfo, _ := devices.lookupDevice("")
+	utils.Debugf("oldInfo: %p", oldInfo)
 	if oldInfo != nil && oldInfo.Initialized {
 		return nil
 	}
@@ -349,7 +398,7 @@ func (devices *DeviceSet) setupBaseImage() error {
 	}
 
 	info.Initialized = true
-	if err = devices.saveMetadata(); err != nil {
+	if err = devices.saveMetadata(info); err != nil {
 		info.Initialized = false
 		utils.Debugf("\n--->Err: %s\n", err)
 		return err
@@ -457,29 +506,7 @@ func (devices *DeviceSet) ResizePool(size int64) error {
 func (devices *DeviceSet) initDevmapper(doInit bool) error {
 	logInit(devices)
 
-	// Make sure the sparse images exist in <root>/devicemapper/data and
+	if err := osMkdirAll(devices.metadataDir(), 0700); err != nil && !osIsExist(err) {
-	// <root>/devicemapper/metadata
-
-	hasData := devices.hasImage("data")
-	hasMetadata := devices.hasImage("metadata")
-
-	if !doInit && !hasData {
-		return errors.New("Loopback data file not found")
-	}
-
-	if !doInit && !hasMetadata {
-		return errors.New("Loopback metadata file not found")
-	}
-
-	createdLoopback := !hasData || !hasMetadata
-	data, err := devices.ensureImage("data", DefaultDataLoopbackSize)
-	if err != nil {
-		utils.Debugf("Error device ensureImage (data): %s\n", err)
-		return err
-	}
-	metadata, err := devices.ensureImage("metadata", DefaultMetaDataLoopbackSize)
-	if err != nil {
-		utils.Debugf("Error device ensureImage (metadata): %s\n", err)
 		return err
 	}
 
@@ -512,10 +539,38 @@ func (devices *DeviceSet) initDevmapper(doInit bool) error {
 	// so we add this badhack to make sure it closes itself
 	setCloseOnExec("/dev/mapper/control")
 
+	// Make sure the sparse images exist in <root>/devicemapper/data and
+	// <root>/devicemapper/metadata
+
+	createdLoopback := false
+
 	// If the pool doesn't exist, create it
 	if info.Exists == 0 {
 		utils.Debugf("Pool doesn't exist. Creating it.")
 
+		hasData := devices.hasImage("data")
+		hasMetadata := devices.hasImage("metadata")
+
+		if !doInit && !hasData {
+			return errors.New("Loopback data file not found")
+		}
+
+		if !doInit && !hasMetadata {
+			return errors.New("Loopback metadata file not found")
+		}
+
+		createdLoopback = !hasData || !hasMetadata
+		data, err := devices.ensureImage("data", DefaultDataLoopbackSize)
+		if err != nil {
+			utils.Debugf("Error device ensureImage (data): %s\n", err)
+			return err
+		}
+		metadata, err := devices.ensureImage("metadata", DefaultMetaDataLoopbackSize)
+		if err != nil {
+			utils.Debugf("Error device ensureImage (metadata): %s\n", err)
+			return err
+		}
+
 		dataFile, err := attachLoopDevice(data)
 		if err != nil {
 			utils.Debugf("\n--->Err: %s\n", err)
@@ -537,9 +592,9 @@ func (devices *DeviceSet) initDevmapper(doInit bool) error {
 	}
 
 	// If we didn't just create the data or metadata image, we need to
-	// load the metadata from the existing file.
+	// load the transaction id and migrate old metadata
 	if !createdLoopback {
-		if err = devices.loadMetaData(); err != nil {
+		if err = devices.initMetaData(); err != nil {
 			utils.Debugf("\n--->Err: %s\n", err)
 			return err
 		}
@@ -608,14 +663,6 @@ func (devices *DeviceSet) deleteDevice(info *DevInfo) error {
 		}
 	}
 
-	if info.Initialized {
-		info.Initialized = false
-		if err := devices.saveMetadata(); err != nil {
-			utils.Debugf("Error saving meta data: %s\n", err)
-			return err
-		}
-	}
-
 	if err := deleteDevice(devices.getPoolDevName(), info.DeviceId); err != nil {
 		utils.Debugf("Error deleting device: %s\n", err)
 		return err
@@ -626,11 +673,11 @@ func (devices *DeviceSet) deleteDevice(info *DevInfo) error {
 	delete(devices.Devices, info.Hash)
 	devices.devicesLock.Unlock()
 
-	if err := devices.saveMetadata(); err != nil {
+	if err := devices.removeMetadata(info); err != nil {
 		devices.devicesLock.Lock()
 		devices.Devices[info.Hash] = info
 		devices.devicesLock.Unlock()
-		utils.Debugf("Error saving meta data: %s\n", err)
+		utils.Debugf("Error removing meta data: %s\n", err)
 		return err
 	}
 
@@ -873,7 +920,7 @@ func (devices *DeviceSet) MountDevice(hash, path string, mountLabel string) erro
 	info.mountCount = 1
 	info.mountPath = path
 
-	return devices.setInitialized(info)
+	return nil
 }
 
 func (devices *DeviceSet) UnmountDevice(hash string) error {
@@ -924,14 +971,6 @@ func (devices *DeviceSet) HasDevice(hash string) bool {
 	return info != nil
 }
 
-func (devices *DeviceSet) HasInitializedDevice(hash string) bool {
-	devices.Lock()
-	defer devices.Unlock()
-
-	info, _ := devices.lookupDevice(hash)
-	return info != nil && info.Initialized
-}
-
 func (devices *DeviceSet) HasActivatedDevice(hash string) bool {
 	info, _ := devices.lookupDevice(hash)
 	if info == nil {
@@ -948,17 +987,6 @@ func (devices *DeviceSet) HasActivatedDevice(hash string) bool {
 	return devinfo != nil && devinfo.Exists != 0
 }
 
-func (devices *DeviceSet) setInitialized(info *DevInfo) error {
-	info.Initialized = true
-	if err := devices.saveMetadata(); err != nil {
-		info.Initialized = false
-		utils.Debugf("\n--->Err: %s\n", err)
-		return err
-	}
-
-	return nil
-}
-
 func (devices *DeviceSet) List() []string {
 	devices.Lock()
 	defer devices.Unlock()

daemon/graphdriver/devmapper/driver_test.go

@@ -820,10 +820,6 @@ func TestGetReturnsValidDevice(t *testing.T) {
 	if !d.HasActivatedDevice("1") {
 		t.Fatalf("Expected id 1 to be activated")
 	}
-
-	if !d.HasInitializedDevice("1") {
-		t.Fatalf("Expected id 1 to be initialized")
-	}
 }
 
 func TestDriverGetSize(t *testing.T) {