Merge remove-hijack

api_test.go

@@ -679,10 +679,82 @@ func TestPostImagesCreate(t *testing.T) {
 	// })
 }
 
-// func TestPostImagesInsert(t *testing.T) {
-// 	//FIXME: Implement this test (or remove this endpoint)
-// 	t.Log("Test not implemented")
-// }
+func TestPostImagesInsert(t *testing.T) {
+	// runtime, err := newTestRuntime()
+	// if err != nil {
+	// 	t.Fatal(err)
+	// }
+	// defer nuke(runtime)
+
+	// srv := &Server{runtime: runtime}
+
+	// stdin, stdinPipe := io.Pipe()
+	// stdout, stdoutPipe := io.Pipe()
+
+	// // Attach to it
+	// c1 := make(chan struct{})
+	// go func() {
+	// 	defer close(c1)
+	// 	r := &hijackTester{
+	// 		ResponseRecorder: httptest.NewRecorder(),
+	// 		in:               stdin,
+	// 		out:              stdoutPipe,
+	// 	}
+
+	// 	req, err := http.NewRequest("POST", "/images/"+unitTestImageName+"/insert?path=%2Ftest&url=https%3A%2F%2Fraw.github.com%2Fdotcloud%2Fdocker%2Fmaster%2FREADME.md", bytes.NewReader([]byte{}))
+	// 	if err != nil {
+	// 		t.Fatal(err)
+	// 	}
+	// 	if err := postContainersCreate(srv, r, req, nil); err != nil {
+	// 		t.Fatal(err)
+	// 	}
+	// }()
+
+	// // Acknowledge hijack
+	// setTimeout(t, "hijack acknowledge timed out", 5*time.Second, func() {
+	// 	stdout.Read([]byte{})
+	// 	stdout.Read(make([]byte, 4096))
+	// })
+
+	// id := ""
+	// setTimeout(t, "Waiting for imagesInsert output", 10*time.Second, func() {
+	// 	for {
+	// 		reader := bufio.NewReader(stdout)
+	// 		id, err = reader.ReadString('\n')
+	// 		if err != nil {
+	// 			t.Fatal(err)
+	// 		}
+	// 	}
+	// })
+
+	// // Close pipes (client disconnects)
+	// if err := closeWrap(stdin, stdinPipe, stdout, stdoutPipe); err != nil {
+	// 	t.Fatal(err)
+	// }
+
+	// // Wait for attach to finish, the client disconnected, therefore, Attach finished his job
+	// setTimeout(t, "Waiting for CmdAttach timed out", 2*time.Second, func() {
+	// 	<-c1
+	// })
+
+	// img, err := srv.runtime.repositories.LookupImage(id)
+	// if err != nil {
+	// 	t.Fatalf("New image %s expected but not found", id)
+	// }
+
+	// layer, err := img.layer()
+	// if err != nil {
+	// 	t.Fatal(err)
+	// }
+
+	// if _, err := os.Stat(path.Join(layer, "test")); err != nil {
+	// 	t.Fatalf("The test file has not been found")
+	// }
+
+	// if err := srv.runtime.graph.Delete(img.Id); err != nil {
+	// 	t.Fatal(err)
+	// }
+}
 
 func TestPostImagesPush(t *testing.T) {
 	//FIXME: Use staging in order to perform tests

commands.go

@@ -104,7 +104,7 @@ func (cli *DockerCli) CmdInsert(args ...string) error {
 	v.Set("url", cmd.Arg(1))
 	v.Set("path", cmd.Arg(2))
 
-	err := cli.stream("POST", "/images/"+cmd.Arg(0)+"?"+v.Encode(), nil, os.Stdout)
+	err := cli.stream("POST", "/images/"+cmd.Arg(0)+"/insert?"+v.Encode(), nil, os.Stdout)
 	if err != nil {
 		return err
 	}

docs/Makefile

@@ -59,18 +59,13 @@ site:
 connect:
 	@echo connecting dotcloud to www.docker.io website, make sure to use user 1
 	@cd _build/website/ ; \
-	dotcloud list ; \
-	dotcloud connect dockerwebsite
+	dotcloud connect dockerwebsite ;
+	dotcloud list
 
 push:
 	@cd _build/website/ ; \
 	dotcloud push
 
-github-deploy: docs
-	rm -fr github-deploy
-	git clone ssh://git@github.com/dotcloud/docker github-deploy
-	cd github-deploy && git checkout -f gh-pages && git rm -r * && rsync -avH ../_build/html/ ./ && touch .nojekyll && echo "docker.io" > CNAME && git add * && git commit -m "Updating docs"
-
 $(VERSIONS):
 	@echo "Hello world"
 

docs/sources/api/docker_remote_api.rst

@@ -9,7 +9,7 @@ Docker Remote API
 
 - The Remote API is replacing rcli
 - Default port in the docker deamon is 4243 
-- The API tends to be REST, but for some complex commands, like attach or pull, the HTTP connection in hijacked to transport stdout stdin and stderr
+- The API tends to be REST, but for some complex commands, like attach or pull, the HTTP connection is hijacked to transport stdout stdin and stderr
 
 2. Endpoints
 ============

docs/sources/api/registry_api.rst

@@ -85,7 +85,7 @@ On top of being a runtime for LXC, Docker is the Registry client. It supports:
 5. Index returns true/false lettings registry know if it should proceed or error out
 6. Get the payload for all layers
 
-It’s possible to run docker pull https://<registry>/repositories/samalba/busybox. In this case, docker bypasses the Index. However the security is not guaranteed (in case Registry A is corrupted) because there won’t be any checksum checks.
+It’s possible to run docker pull \https://<registry>/repositories/samalba/busybox. In this case, docker bypasses the Index. However the security is not guaranteed (in case Registry A is corrupted) because there won’t be any checksum checks.
 
 Currently registry redirects to s3 urls for downloads, going forward all downloads need to be streamed through the registry. The Registry will then abstract the calls to S3 by a top-level class which implements sub-classes for S3 and local storage.
 
@@ -245,8 +245,8 @@ API (pushing repos foo/bar):
 The Index has two main purposes (along with its fancy social features):
 
 - Resolve short names (to avoid passing absolute URLs all the time)
-   - username/projectname -> https://registry.docker.io/users/<username>/repositories/<projectname>/
-   - team/projectname -> https://registry.docker.io/team/<team>/repositories/<projectname>/
+   - username/projectname -> \https://registry.docker.io/users/<username>/repositories/<projectname>/
+   - team/projectname -> \https://registry.docker.io/team/<team>/repositories/<projectname>/
 - Authenticate a user as a repos owner (for a central referenced repository)
 
 3.1 Without an Index

docs/sources/examples/python_web_app.rst

@@ -58,7 +58,7 @@ Use the new image we just created and create a new container with network port 5
 .. code-block:: bash
 
     docker logs $WEB_WORKER
-     * Running on http://0.0.0.0:5000/
+     * Running on \http://0.0.0.0:5000/
 
 view the logs for the new container using the WEB_WORKER variable, and if everything worked as planned you should see the line "Running on http://0.0.0.0:5000/" in the log output.
 
@@ -70,7 +70,7 @@ lookup the public-facing port which is NAT-ed store the private port used by the
 
 .. code-block:: bash
 
-    curl http://`hostname`:$WEB_PORT
+    curl \http://`hostname`:$WEB_PORT
       Hello world!
 
 access the web app using curl. If everything worked as planned you should see the line "Hello world!" inside of your console.

docs/sources/faq.rst

@@ -15,7 +15,7 @@ Most frequently asked questions.
 
 3. **Does Docker run on Mac OS X or Windows?**
 
-   Not at this time, Docker currently only runs on Linux, but you can use VirtualBox to run Docker in a virtual machine on your box, and get the best of both worlds. Check out the MacOSX_ and Windows_ intallation guides.
+   Not at this time, Docker currently only runs on Linux, but you can use VirtualBox to run Docker in a virtual machine on your box, and get the best of both worlds. Check out the MacOSX_ and Windows_ installation guides.
 
 4. **How do containers compare to virtual machines?**
 
@@ -35,8 +35,8 @@ Most frequently asked questions.
     * `Ask questions on Stackoverflow`_
     * `Join the conversation on Twitter`_
 
-    .. _Windows: ../documentation/installation/windows.html
-    .. _MacOSX: ../documentation/installation/macos.html
+    .. _Windows: ../installation/windows/
+    .. _MacOSX: ../installation/vagrant/
     .. _the repo: http://www.github.com/dotcloud/docker
     .. _IRC\: docker on freenode: irc://chat.freenode.net#docker
     .. _Github: http://www.github.com/dotcloud/docker

docs/sources/installation/index.rst

@@ -20,3 +20,4 @@ Contents:
    rackspace
    archlinux
    upgrading
+   kernel

docs/sources/installation/kernel.rst

@@ -0,0 +1,149 @@
+.. _kernel:
+
+Kernel Requirements
+===================
+
+  The officially supported kernel is the one recommended by the
+  :ref:`ubuntu_linux` installation path. It is the one that most developers
+  will use, and the one that receives the most attention from the core
+  contributors. If you decide to go with a different kernel and hit a bug,
+  please try to reproduce it with the official kernels first.
+
+If for some reason you cannot or do not want to use the "official" kernels,
+here is some technical background about the features (both optional and
+mandatory) that docker needs to run successfully.
+
+In short, you need kernel version 3.8 (or above), compiled to include
+`AUFS support <http://aufs.sourceforge.net/>`_. Of course, you need to
+enable cgroups and namespaces.
+
+
+Namespaces and Cgroups
+----------------------
+
+You need to enable namespaces and cgroups, to the extend of what is needed
+to run LXC containers. Technically, while namespaces have been introduced
+in the early 2.6 kernels, we do not advise to try any kernel before 2.6.32
+to run LXC containers. Note that 2.6.32 has some documented issues regarding
+network namespace setup and teardown; those issues are not a risk if you
+run containers in a private environment, but can lead to denial-of-service
+attacks if you want to run untrusted code in your containers. For more details,
+see `[LP#720095 <https://bugs.launchpad.net/ubuntu/+source/linux/+bug/720095>`_.
+
+Kernels 2.6.38, and every version since 3.2, have been deployed successfully
+to run containerized production workloads. Feature-wise, there is no huge
+improvement between 2.6.38 and up to 3.6 (as far as docker is concerned!).
+
+Starting with version 3.7, the kernel has basic support for
+`Checkpoint/Restore In Userspace <http://criu.org/>`_, which is not used by
+docker at this point, but allows to suspend the state of a container to
+disk and resume it later.
+
+Version 3.8 provides improvements in stability, which are deemed necessary
+for the operation of docker. Versions 3.2 to 3.5 have been shown to
+exhibit a reproducible bug (for more details, see issue
+`#407 <https://github.com/dotcloud/docker/issues/407>`_).
+
+Version 3.8 also brings better support for the
+`setns() syscall <http://lwn.net/Articles/531381/>`_ -- but this should not
+be a concern since docker does not leverage on this feature for now.
+
+If you want a technical overview about those concepts, you might
+want to check those articles on dotCloud's blog:
+`about namespaces <http://blog.dotcloud.com/under-the-hood-linux-kernels-on-dotcloud-part>`_
+and `about cgroups <http://blog.dotcloud.com/kernel-secrets-from-the-paas-garage-part-24-c>`_.
+
+
+Important Note About Pre-3.8 Kernels
+------------------------------------
+
+As mentioned above, kernels before 3.8 are not stable when used with docker.
+In some circumstances, you will experience kernel "oopses", or even crashes.
+The symptoms include:
+
+- a container being killed in the middle of an operation (e.g. an ``apt-get``
+  command doesn't complete);
+- kernel messages including mentioning calls to ``mntput`` or
+  ``d_hash_and_lookup``;
+- kernel crash causing the machine to freeze for a few minutes, or even
+  completely.
+
+While it is still possible to use older kernels for development, it is
+really not advised to do so.
+
+Docker checks the kernel version when it starts, and emits a warning if it
+detects something older than 3.8.
+
+See issue `#407 <https://github.com/dotcloud/docker/issues/407>`_ for details.
+
+
+Extra Cgroup Controllers
+------------------------
+
+Most control groups can be enabled or disabled individually. For instance,
+you can decide that you do not want to compile support for the CPU or memory
+controller. In some cases, the feature can be enabled or disabled at boot
+time. It is worth mentioning that some distributions (like Debian) disable
+"expensive" features, like the memory controller, because they can have
+a significant performance impact.
+
+In the specific case of the memory cgroup, docker will detect if the cgroup
+is available or not. If it's not, it will print a warning, and it won't
+use the feature. If you want to enable that feature -- read on!
+
+
+Memory and Swap Accounting on Debian/Ubuntu
+-------------------------------------------
+
+If you use Debian or Ubuntu kernels, and want to enable memory and swap
+accounting, you must add the following command-line parameters to your kernel::
+
+    cgroup_enable=memory swapaccount
+
+On Debian or Ubuntu systems, if you use the default GRUB bootloader, you can
+add those parameters by editing ``/etc/default/grub`` and extending
+``GRUB_CMDLINE_LINUX``. Look for the following line::
+
+    GRUB_CMDLINE_LINUX=""
+
+And replace it by the following one::
+
+    GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount"
+
+Then run ``update-grub``, and reboot.
+
+
+AUFS
+----
+
+Docker currently relies on AUFS, an unioning filesystem.
+While AUFS is included in the kernels built by the Debian and Ubuntu
+distributions, is not part of the standard kernel. This means that if
+you decide to roll your own kernel, you will have to patch your
+kernel tree to add AUFS. The process is documented on
+`AUFS webpage <http://aufs.sourceforge.net/>`_.
+
+Note: the AUFS patch is fairly intrusive, but for the record, people have
+successfully applied GRSEC and AUFS together, to obtain hardened production
+kernels.
+
+If you want more information about that topic, there is an
+`article about AUFS on dotCloud's blog 
+<http://blog.dotcloud.com/kernel-secrets-from-the-paas-garage-part-34-a>`_.
+
+
+BTRFS, ZFS, OverlayFS...
+------------------------
+
+There is ongoing development on docker, to implement support for
+`BTRFS <http://en.wikipedia.org/wiki/Btrfs>`_
+(see github issue `#443 <https://github.com/dotcloud/docker/issues/443>`_).
+
+People have also showed interest for `ZFS <http://en.wikipedia.org/wiki/ZFS>`_
+(using e.g. `ZFS-on-Linux <http://zfsonlinux.org/>`_) and OverlayFS.
+The latter is functionally close to AUFS, and it might end up being included
+in the stock kernel; so it's a strong candidate!
+
+Would you like to `contribute
+<https://github.com/dotcloud/docker/blob/master/CONTRIBUTING.md>`_
+support for your favorite filesystem?

docs/sources/use/index.rst

@@ -15,4 +15,5 @@ Contents:
    basics
    workingwithrepository
    builder
+   puppet
 

docs/sources/use/puppet.rst

@@ -0,0 +1,109 @@
+
+.. _install_using_puppet:
+
+Using Puppet
+=============
+
+.. note::
+
+   Please note this is a community contributed installation path. The only 'official' installation is using the
+   :ref:`ubuntu_linux` installation path. This version may sometimes be out of date.
+
+Requirements
+------------
+
+To use this guide you'll need a working installation of Puppet from `Puppetlabs <https://www.puppetlabs.com>`_ .
+
+The module also currently uses the official PPA so only works with Ubuntu.
+
+Installation
+------------
+
+The module is available on the `Puppet Forge <https://forge.puppetlabs.com/garethr/docker/>`_
+and can be installed using the built-in module tool.
+
+   .. code-block:: bash
+
+      puppet module install garethr/docker
+
+It can also be found on `GitHub <https://www.github.com/garethr/garethr-docker>`_ 
+if you would rather download the source.
+
+Usage
+-----
+
+The module provides a puppet class for installing docker and two defined types
+for managing images and containers.
+
+Installation
+~~~~~~~~~~~~
+
+   .. code-block:: ruby
+
+      include 'docker'
+
+Images
+~~~~~~
+
+The next step is probably to install a docker image, for this we have a
+defined type which can be used like so:
+
+   .. code-block:: ruby
+
+      docker::image { 'base': }
+
+This is equivalent to running:
+
+   .. code-block:: bash
+
+      docker pull base
+
+Note that it will only if the image of that name does not already exist.
+This is downloading a large binary so on first run can take a while.
+For that reason this define turns off the default 5 minute timeout
+for exec. Note that you can also remove images you no longer need with:
+
+   .. code-block:: ruby
+
+      docker::image { 'base':
+        ensure => 'absent',
+      }
+
+Containers
+~~~~~~~~~~
+
+Now you have an image you can run commands within a container managed by
+docker.
+
+   .. code-block:: ruby
+
+      docker::run { 'helloworld':
+        image   => 'base',
+        command => '/bin/sh -c "while true; do echo hello world; sleep 1; done"',
+      }
+
+This is equivalent to running the following command, but under upstart:
+
+   .. code-block:: bash
+
+      docker run -d base /bin/sh -c "while true; do echo hello world; sleep 1; done"
+
+Run also contains a number of optional parameters:
+
+   .. code-block:: ruby
+
+      docker::run { 'helloworld':
+        image        => 'base',
+        command      => '/bin/sh -c "while true; do echo hello world; sleep 1; done"',
+        ports        => ['4444', '4555'],
+        volumes      => ['/var/lib/counchdb', '/var/log'],
+        volumes_from => '6446ea52fbc9',
+        memory_limit => 10485760, # bytes 
+        username     => 'example',
+        hostname     => 'example.com',
+        env          => ['FOO=BAR', 'FOO2=BAR2'],
+        dns          => ['8.8.8.8', '8.8.4.4'],
+      }
+
+Note that ports, env, dns and volumes can be set with either a single string
+or as above with an array of values.

docs/website/index.html

@@ -127,6 +127,22 @@
                 <h4>Repeatability</h4>
                 <p>Because each container is isolated in its own filesystem, they behave the same regardless of where, when, and alongside what they run.</p>
             </section>
+            <section class="contentblock">
+                <div class="container">
+                <div class="span2" style="margin-left: 0" >
+                    <a href="http://dotcloud.theresumator.com/apply/mWjkD4/Software-Engineer.html" title="Job description"><img src="static/img/hiring_graphic.png" width="140px" style="margin-top: 25px"></a>
+                </div>
+                <div class="span4" style="margin-left: 0">
+                    <h4>Do you think it is cool to hack on docker? Join us!</h4>
+                    <ul>
+                        <li>Work on open source</li>
+                        <li>Program in Go</li>
+                    </ul>
+                    <a href="http://dotcloud.theresumator.com/apply/mWjkD4/Software-Engineer.html" title="Job description">read more</a>
+                </div>
+                </div>
+
+            </section>
         </div>
         <div class="span6">
             <section class="contentblock">

server.go

@@ -344,13 +344,24 @@ func (srv *Server) pullRepository(out io.Writer, remote, askedTag string) error
 	if err != nil {
 		return err
 	}
-	for tag, id := range tagsList {
-		repoData.ImgList[id].Tag = tag
+	utils.Debugf("Registering tags")
+	// If not specific tag have been asked, take all
+	if askedTag == "" {
+		for tag, id := range tagsList {
+			repoData.ImgList[id].Tag = tag
+		}
+	} else {
+		// Otherwise, check that the tag exists and use only that one
+		if id, exists := tagsList[askedTag]; !exists {
+			return fmt.Errorf("Tag %s not found in repositoy %s", askedTag, remote)
+		} else {
+			repoData.ImgList[id].Tag = askedTag
+		}
 	}
 
 	for _, img := range repoData.ImgList {
-		// If we asked for a specific tag, skip all tags expect the wanted one
-		if askedTag != "" && askedTag != img.Tag {
+		if askedTag != "" && img.Tag != askedTag {
+			utils.Debugf("%s does not match %s, skipping", img.Tag, askedTag)
 			continue
 		}
 		fmt.Fprintf(out, "Pulling image %s (%s) from %s\n", img.Id, img.Tag, remote)
@@ -371,6 +382,10 @@ func (srv *Server) pullRepository(out io.Writer, remote, askedTag string) error
 			return fmt.Errorf("Could not find repository on any of the indexed registries.")
 		}
 	}
+	// If we asked for a specific tag, do not register the others
+	if askedTag != "" {
+		return nil
+	}
 	for tag, id := range tagsList {
 		if err := srv.runtime.repositories.Set(remote, tag, id, true); err != nil {
 			return err