From 854fe82ba10cad3fef18f4fcd42122b6a05a9852 Mon Sep 17 00:00:00 2001 From: Alessandro Boch Date: Mon, 11 Jan 2016 13:03:52 -0800 Subject: [PATCH] Allow bitseq caller to run consistency check Signed-off-by: Alessandro Boch --- libnetwork/bitseq/sequence.go | 53 +++++++++++ libnetwork/bitseq/sequence_test.go | 148 +++++++++++++++++++++++++++++ 2 files changed, 201 insertions(+) diff --git a/libnetwork/bitseq/sequence.go b/libnetwork/bitseq/sequence.go index a537ed0107..270a36aa63 100644 --- a/libnetwork/bitseq/sequence.go +++ b/libnetwork/bitseq/sequence.go @@ -9,6 +9,7 @@ import ( "fmt" "sync" + log "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/datastore" "github.com/docker/libnetwork/types" ) @@ -243,6 +244,58 @@ func (h *Handle) IsSet(ordinal uint64) bool { return err != nil } +func (h *Handle) runConsistencyCheck() bool { + corrupted := false + for p, c := h.head, h.head.next; c != nil; c = c.next { + if c.count == 0 { + corrupted = true + p.next = c.next + continue // keep same p + } + p = c + } + return corrupted +} + +// CheckConsistency checks if the bit sequence is in an inconsistent state and attempts to fix it. +// It looks for a corruption signature that may happen in docker 1.9.0 and 1.9.1. +func (h *Handle) CheckConsistency() error { + for { + h.Lock() + store := h.store + h.Unlock() + + if store != nil { + if err := store.GetObject(datastore.Key(h.Key()...), h); err != nil && err != datastore.ErrKeyNotFound { + return err + } + } + + h.Lock() + nh := h.getCopy() + h.Unlock() + + if !nh.runConsistencyCheck() { + return nil + } + + if err := nh.writeToStore(); err != nil { + if _, ok := err.(types.RetryError); !ok { + return fmt.Errorf("internal failure while fixing inconsistent bitsequence: %v", err) + } + continue + } + + log.Infof("Fixed inconsistent bit sequence in datastore:\n%s\n%s", h, nh) + + h.Lock() + h.head = nh.head + h.Unlock() + + return nil + } +} + // set/reset the bit func (h *Handle) set(ordinal, start, end uint64, any bool, release bool) (uint64, error) { var ( diff --git a/libnetwork/bitseq/sequence_test.go b/libnetwork/bitseq/sequence_test.go index 54c0b86cd6..dff1177fbc 100644 --- a/libnetwork/bitseq/sequence_test.go +++ b/libnetwork/bitseq/sequence_test.go @@ -980,3 +980,151 @@ func TestRetrieveFromStore(t *testing.T) { t.Fatal(err) } } + +func TestIsCorrupted(t *testing.T) { + ds, err := randomLocalStore() + if err != nil { + t.Fatal(err) + } + // Negative test + hnd, err := NewHandle("bitseq-test/data/", ds, "test_corrupted", 1024) + if err != nil { + t.Fatal(err) + } + + if hnd.runConsistencyCheck() { + t.Fatalf("Unexpected corrupted for %s", hnd) + } + + if err := hnd.CheckConsistency(); err != nil { + t.Fatal(err) + } + + hnd.Set(0) + if hnd.runConsistencyCheck() { + t.Fatalf("Unexpected corrupted for %s", hnd) + } + + hnd.Set(1023) + if hnd.runConsistencyCheck() { + t.Fatalf("Unexpected corrupted for %s", hnd) + } + + if err := hnd.CheckConsistency(); err != nil { + t.Fatal(err) + } + + // Try real corrupted ipam handles found in the local store files reported by three docker users, + // plus a generic ipam handle from docker 1.9.1. This last will fail as well, because of how the + // last node in the sequence is expressed (This is true for IPAM handle only, because of the broadcast + // address reservation: last bit). This will allow an application using bitseq that runs a consistency + // check to detect and replace the 1.9.0/1 old vulnerable handle with the new one. + input := []*Handle{ + &Handle{ + id: "LocalDefault/172.17.0.0/16", + bits: 65536, + unselected: 65412, + head: &sequence{ + block: 0xffffffff, + count: 3, + next: &sequence{ + block: 0xffffffbf, + count: 0, + next: &sequence{ + block: 0xfe98816e, + count: 1, + next: &sequence{ + block: 0xffffffff, + count: 0, + next: &sequence{ + block: 0xe3bc0000, + count: 1, + next: &sequence{ + block: 0x0, + count: 2042, + next: &sequence{ + block: 0x1, count: 1, + next: &sequence{ + block: 0x0, count: 0, + }, + }, + }, + }, + }, + }, + }, + }, + }, + &Handle{ + id: "LocalDefault/172.17.0.0/16", + bits: 65536, + unselected: 65319, + head: &sequence{ + block: 0xffffffff, + count: 7, + next: &sequence{ + block: 0xffffff7f, + count: 0, + next: &sequence{ + block: 0xffffffff, + count: 0, + next: &sequence{ + block: 0x2000000, + count: 1, + next: &sequence{ + block: 0x0, + count: 2039, + next: &sequence{ + block: 0x1, + count: 1, + next: &sequence{ + block: 0x0, + count: 0, + }, + }, + }, + }, + }, + }, + }, + }, + &Handle{ + id: "LocalDefault/172.17.0.0/16", + bits: 65536, + unselected: 65456, + head: &sequence{ + block: 0xffffffff, count: 2, + next: &sequence{ + block: 0xfffbffff, count: 0, + next: &sequence{ + block: 0xffd07000, count: 1, + next: &sequence{ + block: 0x0, count: 333, + next: &sequence{ + block: 0x40000000, count: 1, + next: &sequence{ + block: 0x0, count: 1710, + next: &sequence{ + block: 0x1, count: 1, + next: &sequence{ + block: 0x0, count: 0, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + for idx, hnd := range input { + if !hnd.runConsistencyCheck() { + t.Fatalf("Expected corrupted for (%d): %s", idx, hnd) + } + if hnd.runConsistencyCheck() { + t.Fatalf("Sequence still marked corrupted (%d): %s", idx, hnd) + } + } +}