godeps: Add github.com/cznic/ql.

Godeps/Godeps.json

@@ -47,6 +47,10 @@
 			"ImportPath": "github.com/coreos/go-iptables/iptables",
 			"Rev": "83dfad0f13fd7310fb3c1cb8563248d8d604b95b"
 		},
+		{
+			"ImportPath": "github.com/camlistore/lock",
+			"Rev": "ae27720f340952636b826119b58130b9c1a847a0"
+		},
 		{
 			"ImportPath": "github.com/coreos/go-semver/semver",
 			"Rev": "6fe83ccda8fb9b7549c9ab4ba47f47858bc950aa"
@@ -56,6 +60,38 @@
 			"Comment": "v2-41-g415a79c4bfa3",
 			"Rev": "415a79c4bfa377b484b16c632347b3925a0f35ff"
 		},
+		{
+			"ImportPath": "github.com/cznic/bufs",
+			"Rev": "3dcccbd7064a1689f9c093a988ea11ac00e21f51"
+		},
+		{
+			"ImportPath": "github.com/cznic/exp/lldb",
+			"Rev": "9b0e4be12fbdb7b843e0a658a04c35d160371789"
+		},
+		{
+			"ImportPath": "github.com/cznic/fileutil",
+			"Rev": "21ae57c9dce724a15e88bd9cd46d5668f3e880a5"
+		},
+		{
+			"ImportPath": "github.com/cznic/mathutil",
+			"Rev": "250d0b9d3304c5ea0c4cfc7d9efc7ee528b81f3b"
+		},
+		{
+			"ImportPath": "github.com/cznic/ql",
+			"Rev": "fc1b91b82089d3f132fbed8a7c9f349c3133eb96"
+		},
+		{
+			"ImportPath": "github.com/cznic/sortutil",
+			"Rev": "d4401851b4c370f979b842fa1e45e0b3b718b391"
+		},
+		{
+			"ImportPath": "github.com/cznic/strutil",
+			"Rev": "97bc31f80ac4c9fa9c5dc5fea74c383858988ea2"
+		},
+		{
+			"ImportPath": "github.com/cznic/zappy",
+			"Rev": "47331054e4f96186e3ff772877c0443909368a45"
+		},
 		{
 			"ImportPath": "github.com/gorilla/context",
 			"Rev": "50c25fb3b2b3b3cc724e9b6ac75fb44b3bccd0da"

Godeps/_workspace/src/github.com/camlistore/lock/.gitignore

@@ -0,0 +1 @@
+*~

Godeps/_workspace/src/github.com/camlistore/lock/COPYING

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

Godeps/_workspace/src/github.com/camlistore/lock/README.txt

@@ -0,0 +1,3 @@
+File locking library.
+
+See http://godoc.org/github.com/camlistore/lock

Godeps/_workspace/src/github.com/camlistore/lock/lock.go

@@ -0,0 +1,158 @@
+/*
+Copyright 2013 The Go Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package lock
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"sync"
+)
+
+// Lock locks the given file, creating the file if necessary. If the
+// file already exists, it must have zero size or an error is returned.
+// The lock is an exclusive lock (a write lock), but locked files
+// should neither be read from nor written to. Such files should have
+// zero size and only exist to co-ordinate ownership across processes.
+//
+// A nil Closer is returned if an error occurred. Otherwise, close that
+// Closer to release the lock.
+//
+// On Linux, FreeBSD and OSX, a lock has the same semantics as fcntl(2)'s
+// advisory locks.  In particular, closing any other file descriptor for the
+// same file will release the lock prematurely.
+//
+// Attempting to lock a file that is already locked by the current process
+// has undefined behavior.
+//
+// On other operating systems, lock will fallback to using the presence and
+// content of a file named name + '.lock' to implement locking behavior.
+func Lock(name string) (io.Closer, error) {
+	return lockFn(name)
+}
+
+var lockFn = lockPortable
+
+// Portable version not using fcntl. Doesn't handle crashes as gracefully,
+// since it can leave stale lock files.
+// TODO: write pid of owner to lock file and on race see if pid is
+// still alive?
+func lockPortable(name string) (io.Closer, error) {
+	absName, err := filepath.Abs(name)
+	if err != nil {
+		return nil, fmt.Errorf("can't Lock file %q: can't find abs path: %v", name, err)
+	}
+	fi, err := os.Stat(absName)
+	if err == nil && fi.Size() > 0 {
+		if isStaleLock(absName) {
+			os.Remove(absName)
+		} else {
+			return nil, fmt.Errorf("can't Lock file %q: has non-zero size", name)
+		}
+	}
+	f, err := os.OpenFile(absName, os.O_RDWR|os.O_CREATE|os.O_TRUNC|os.O_EXCL, 0666)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create lock file %s %v", absName, err)
+	}
+	if err := json.NewEncoder(f).Encode(&pidLockMeta{OwnerPID: os.Getpid()}); err != nil {
+		return nil, err
+	}
+	return &lockCloser{f: f, abs: absName}, nil
+}
+
+type pidLockMeta struct {
+	OwnerPID int
+}
+
+func isStaleLock(path string) bool {
+	f, err := os.Open(path)
+	if err != nil {
+		return false
+	}
+	defer f.Close()
+	var meta pidLockMeta
+	if json.NewDecoder(f).Decode(&meta) != nil {
+		return false
+	}
+	if meta.OwnerPID == 0 {
+		return false
+	}
+	p, err := os.FindProcess(meta.OwnerPID)
+	if err != nil {
+		// e.g. on Windows
+		return true
+	}
+	// On unix, os.FindProcess always is true, so we have to send
+	// it a signal to see if it's alive.
+	if signalZero != nil {
+		if p.Signal(signalZero) != nil {
+			return true
+		}
+	}
+	return false
+}
+
+var signalZero os.Signal // nil or set by lock_sigzero.go
+
+type lockCloser struct {
+	f    *os.File
+	abs  string
+	once sync.Once
+	err  error
+}
+
+func (lc *lockCloser) Close() error {
+	lc.once.Do(lc.close)
+	return lc.err
+}
+
+func (lc *lockCloser) close() {
+	if err := lc.f.Close(); err != nil {
+		lc.err = err
+	}
+	if err := os.Remove(lc.abs); err != nil {
+		lc.err = err
+	}
+}
+
+var (
+	lockmu sync.Mutex
+	locked = map[string]bool{} // abs path -> true
+)
+
+// unlocker is used by the darwin and linux implementations with fcntl
+// advisory locks.
+type unlocker struct {
+	f   *os.File
+	abs string
+}
+
+func (u *unlocker) Close() error {
+	lockmu.Lock()
+	// Remove is not necessary but it's nice for us to clean up.
+	// If we do do this, though, it needs to be before the
+	// u.f.Close below.
+	os.Remove(u.abs)
+	if err := u.f.Close(); err != nil {
+		return err
+	}
+	delete(locked, u.abs)
+	lockmu.Unlock()
+	return nil
+}

Godeps/_workspace/src/github.com/camlistore/lock/lock_appengine.go

@@ -0,0 +1,32 @@
+// +build appengine
+
+/*
+Copyright 2013 The Go Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package lock
+
+import (
+	"errors"
+	"io"
+)
+
+func init() {
+	lockFn = lockAppEngine
+}
+
+func lockAppEngine(name string) (io.Closer, error) {
+	return nil, errors.New("Lock not available on App Engine")
+}

Godeps/_workspace/src/github.com/camlistore/lock/lock_darwin_amd64.go

@@ -0,0 +1,80 @@
+// +build darwin,amd64
+// +build !appengine
+
+/*
+Copyright 2013 The Go Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package lock
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"syscall"
+	"unsafe"
+)
+
+func init() {
+	lockFn = lockFcntl
+}
+
+func lockFcntl(name string) (io.Closer, error) {
+	abs, err := filepath.Abs(name)
+	if err != nil {
+		return nil, err
+	}
+	lockmu.Lock()
+	if locked[abs] {
+		lockmu.Unlock()
+		return nil, fmt.Errorf("file %q already locked", abs)
+	}
+	locked[abs] = true
+	lockmu.Unlock()
+
+	fi, err := os.Stat(name)
+	if err == nil && fi.Size() > 0 {
+		return nil, fmt.Errorf("can't Lock file %q: has non-zero size", name)
+	}
+
+	f, err := os.Create(name)
+	if err != nil {
+		return nil, fmt.Errorf("Lock Create of %s (abs: %s) failed: %v", name, abs, err)
+	}
+
+	// This type matches C's "struct flock" defined in /usr/include/sys/fcntl.h.
+	// TODO: move this into the standard syscall package.
+	k := struct {
+		Start  uint64 // sizeof(off_t): 8
+		Len    uint64 // sizeof(off_t): 8
+		Pid    uint32 // sizeof(pid_t): 4
+		Type   uint16 // sizeof(short): 2
+		Whence uint16 // sizeof(short): 2
+	}{
+		Type:   syscall.F_WRLCK,
+		Whence: uint16(os.SEEK_SET),
+		Start:  0,
+		Len:    0, // 0 means to lock the entire file.
+		Pid:    uint32(os.Getpid()),
+	}
+
+	_, _, errno := syscall.Syscall(syscall.SYS_FCNTL, f.Fd(), uintptr(syscall.F_SETLK), uintptr(unsafe.Pointer(&k)))
+	if errno != 0 {
+		f.Close()
+		return nil, errno
+	}
+	return &unlocker{f, abs}, nil
+}

Godeps/_workspace/src/github.com/camlistore/lock/lock_freebsd.go

@@ -0,0 +1,79 @@
+/*
+Copyright 2013 The Go Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package lock
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"syscall"
+	"unsafe"
+)
+
+func init() {
+	lockFn = lockFcntl
+}
+
+func lockFcntl(name string) (io.Closer, error) {
+	abs, err := filepath.Abs(name)
+	if err != nil {
+		return nil, err
+	}
+	lockmu.Lock()
+	if locked[abs] {
+		lockmu.Unlock()
+		return nil, fmt.Errorf("file %q already locked", abs)
+	}
+	locked[abs] = true
+	lockmu.Unlock()
+
+	fi, err := os.Stat(name)
+	if err == nil && fi.Size() > 0 {
+		return nil, fmt.Errorf("can't Lock file %q: has non-zero size", name)
+	}
+
+	f, err := os.Create(name)
+	if err != nil {
+		return nil, err
+	}
+
+	// This type matches C's "struct flock" defined in /usr/include/fcntl.h.
+	// TODO: move this into the standard syscall package.
+	k := struct {
+		Start  int64 /* off_t starting offset */
+		Len    int64 /* off_t len = 0 means until end of file */
+		Pid    int32 /* pid_t lock owner */
+		Type   int16 /* short lock type: read/write, etc. */
+		Whence int16 /* short type of l_start */
+		Sysid  int32 /* int   remote system id or zero for local */
+	}{
+		Start:  0,
+		Len:    0, // 0 means to lock the entire file.
+		Pid:    int32(os.Getpid()),
+		Type:   syscall.F_WRLCK,
+		Whence: int16(os.SEEK_SET),
+		Sysid:  0,
+	}
+
+	_, _, errno := syscall.Syscall(syscall.SYS_FCNTL, f.Fd(), uintptr(syscall.F_SETLK), uintptr(unsafe.Pointer(&k)))
+	if errno != 0 {
+		f.Close()
+		return nil, errno
+	}
+	return &unlocker{f, abs}, nil
+}

Godeps/_workspace/src/github.com/camlistore/lock/lock_linux_amd64.go

@@ -0,0 +1,80 @@
+// +build linux,amd64
+// +build !appengine
+
+/*
+Copyright 2013 The Go Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package lock
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"syscall"
+	"unsafe"
+)
+
+func init() {
+	lockFn = lockFcntl
+}
+
+func lockFcntl(name string) (io.Closer, error) {
+	abs, err := filepath.Abs(name)
+	if err != nil {
+		return nil, err
+	}
+	lockmu.Lock()
+	if locked[abs] {
+		lockmu.Unlock()
+		return nil, fmt.Errorf("file %q already locked", abs)
+	}
+	locked[abs] = true
+	lockmu.Unlock()
+
+	fi, err := os.Stat(name)
+	if err == nil && fi.Size() > 0 {
+		return nil, fmt.Errorf("can't Lock file %q: has non-zero size", name)
+	}
+
+	f, err := os.Create(name)
+	if err != nil {
+		return nil, err
+	}
+
+	// This type matches C's "struct flock" defined in /usr/include/bits/fcntl.h.
+	// TODO: move this into the standard syscall package.
+	k := struct {
+		Type   uint32
+		Whence uint32
+		Start  uint64
+		Len    uint64
+		Pid    uint32
+	}{
+		Type:   syscall.F_WRLCK,
+		Whence: uint32(os.SEEK_SET),
+		Start:  0,
+		Len:    0, // 0 means to lock the entire file.
+		Pid:    uint32(os.Getpid()),
+	}
+
+	_, _, errno := syscall.Syscall(syscall.SYS_FCNTL, f.Fd(), uintptr(syscall.F_SETLK), uintptr(unsafe.Pointer(&k)))
+	if errno != 0 {
+		f.Close()
+		return nil, errno
+	}
+	return &unlocker{f, abs}, nil
+}

Godeps/_workspace/src/github.com/camlistore/lock/lock_linux_arm.go

@@ -0,0 +1,81 @@
+// +build linux,arm
+// +build !appengine
+
+/*
+Copyright 2013 The Go Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package lock
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"syscall"
+	"unsafe"
+)
+
+func init() {
+	lockFn = lockFcntl
+}
+
+func lockFcntl(name string) (io.Closer, error) {
+	abs, err := filepath.Abs(name)
+	if err != nil {
+		return nil, err
+	}
+	lockmu.Lock()
+	if locked[abs] {
+		lockmu.Unlock()
+		return nil, fmt.Errorf("file %q already locked", abs)
+	}
+	locked[abs] = true
+	lockmu.Unlock()
+
+	fi, err := os.Stat(name)
+	if err == nil && fi.Size() > 0 {
+		return nil, fmt.Errorf("can't Lock file %q: has non-zero size", name)
+	}
+
+	f, err := os.Create(name)
+	if err != nil {
+		return nil, err
+	}
+
+	// This type matches C's "struct flock" defined in /usr/include/bits/fcntl.h.
+	// TODO: move this into the standard syscall package.
+	k := struct {
+		Type   uint16
+		Whence uint16
+		Start  uint32
+		Len    uint32
+		Pid    uint32
+	}{
+		Type:   syscall.F_WRLCK,
+		Whence: uint16(os.SEEK_SET),
+		Start:  0,
+		Len:    0, // 0 means to lock the entire file.
+		Pid:    uint32(os.Getpid()),
+	}
+
+	const F_SETLK = 6 // actual value. syscall package is wrong: golang.org/issue/7059
+	_, _, errno := syscall.Syscall(syscall.SYS_FCNTL, f.Fd(), uintptr(F_SETLK), uintptr(unsafe.Pointer(&k)))
+	if errno != 0 {
+		f.Close()
+		return nil, errno
+	}
+	return &unlocker{f, abs}, nil
+}

Godeps/_workspace/src/github.com/camlistore/lock/lock_plan9.go

@@ -0,0 +1,55 @@
+/*
+Copyright 2013 The Go Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package lock
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+)
+
+func init() {
+	lockFn = lockPlan9
+}
+
+func lockPlan9(name string) (io.Closer, error) {
+	var f *os.File
+	abs, err := filepath.Abs(name)
+	if err != nil {
+		return nil, err
+	}
+	lockmu.Lock()
+	if locked[abs] {
+		lockmu.Unlock()
+		return nil, fmt.Errorf("file %q already locked", abs)
+	}
+	locked[abs] = true
+	lockmu.Unlock()
+
+	fi, err := os.Stat(name)
+	if err == nil && fi.Size() > 0 {
+		return nil, fmt.Errorf("can't Lock file %q: has non-zero size", name)
+	}
+
+	f, err = os.OpenFile(name, os.O_RDWR|os.O_CREATE, os.ModeExclusive|0644)
+	if err != nil {
+		return nil, fmt.Errorf("Lock Create of %s (abs: %s) failed: %v", name, abs, err)
+	}
+
+	return &unlocker{f, abs}, nil
+}

Godeps/_workspace/src/github.com/camlistore/lock/lock_sigzero.go

@@ -0,0 +1,26 @@
+// +build !appengine
+// +build linux darwin freebsd openbsd netbsd dragonfly
+
+/*
+Copyright 2013 The Go Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package lock
+
+import "syscall"
+
+func init() {
+	signalZero = syscall.Signal(0)
+}

Godeps/_workspace/src/github.com/camlistore/lock/lock_test.go

@@ -0,0 +1,131 @@
+/*
+Copyright 2013 The Go Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package lock
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"testing"
+)
+
+func TestLock(t *testing.T) {
+	testLock(t, false)
+}
+
+func TestLockPortable(t *testing.T) {
+	testLock(t, true)
+}
+
+func TestLockInChild(t *testing.T) {
+	f := os.Getenv("TEST_LOCK_FILE")
+	if f == "" {
+		// not child
+		return
+	}
+	lock := Lock
+	if v, _ := strconv.ParseBool(os.Getenv("TEST_LOCK_PORTABLE")); v {
+		lock = lockPortable
+	}
+
+	lk, err := lock(f)
+	if err != nil {
+		log.Fatalf("Lock failed: %v", err)
+	}
+
+	if v, _ := strconv.ParseBool(os.Getenv("TEST_LOCK_CRASH")); v {
+		// Simulate a crash, or at least not unlocking the
+		// lock.  We still exit 0 just to simplify the parent
+		// process exec code.
+		os.Exit(0)
+	}
+	lk.Close()
+}
+
+func testLock(t *testing.T, portable bool) {
+	lock := Lock
+	if portable {
+		lock = lockPortable
+	}
+
+	td, err := ioutil.TempDir("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(td)
+
+	path := filepath.Join(td, "foo.lock")
+
+	childLock := func(crash bool) error {
+		cmd := exec.Command(os.Args[0], "-test.run=LockInChild$")
+		cmd.Env = []string{"TEST_LOCK_FILE=" + path}
+		if portable {
+			cmd.Env = append(cmd.Env, "TEST_LOCK_PORTABLE=1")
+		}
+		if crash {
+			cmd.Env = append(cmd.Env, "TEST_LOCK_CRASH=1")
+		}
+		out, err := cmd.CombinedOutput()
+		t.Logf("Child output: %q (err %v)", out, err)
+		if err != nil {
+			return fmt.Errorf("Child Process lock of %s failed: %v %s", path, err, out)
+		}
+		return nil
+	}
+
+	t.Logf("Locking in crashing child...")
+	if err := childLock(true); err != nil {
+		t.Fatalf("first lock in child process: %v", err)
+	}
+
+	t.Logf("Locking+unlocking in child...")
+	if err := childLock(false); err != nil {
+		t.Fatalf("lock in child process after crashing child: %v", err)
+	}
+
+	t.Logf("Locking in parent...")
+	lk1, err := lock(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Logf("Again in parent...")
+	_, err = lock(path)
+	if err == nil {
+		t.Fatal("expected second lock to fail")
+	}
+
+	t.Logf("Locking in child...")
+	if childLock(false) == nil {
+		t.Fatalf("expected lock in child process to fail")
+	}
+
+	t.Logf("Unlocking lock in parent")
+	if err := lk1.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	lk3, err := lock(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	lk3.Close()
+}

Godeps/_workspace/src/github.com/cznic/bufs/AUTHORS

@@ -0,0 +1,11 @@
+# This file lists authors for copyright purposes.  This file is distinct from
+# the CONTRIBUTORS files.  See the latter for an explanation.
+#
+# Names should be added to this file as:
+#     Name or Organization <email address>
+#
+# The email address is not required for organizations.
+#
+# Please keep the list sorted.
+
+Jan Mercl <0xjnml@gmail.com>

Godeps/_workspace/src/github.com/cznic/bufs/CONTRIBUTORS

@@ -0,0 +1,9 @@
+# This file lists people who contributed code to this repository.  The AUTHORS
+# file lists the copyright holders; this file lists people.
+#
+# Names should be added to this file like so:
+#     Name <email address>
+#
+# Please keep the list sorted.
+
+Jan Mercl <0xjnml@gmail.com>

Godeps/_workspace/src/github.com/cznic/bufs/LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2014 The bufs Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the names of the authors nor the names of the
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

Godeps/_workspace/src/github.com/cznic/bufs/Makefile

@@ -0,0 +1,31 @@
+# Copyright 2014 The bufs Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+all: clean
+	go fmt
+	go test -i
+	go test
+	go build
+	go vet
+	golint .
+	go install
+	make todo
+
+todo:
+	@grep -n ^[[:space:]]*_[[:space:]]*=[[:space:]][[:alnum:]] *.go || true
+	@grep -n TODO *.go || true
+	@grep -n FIXME *.go || true
+	@grep -n BUG *.go || true
+
+clean:
+	rm -f bufs.test mem.out *~
+
+demo:
+	go test -bench . -benchmem
+	go test -c
+	./bufs.test -test.v -test.run Foo -test.memprofile mem.out \
+		-test.memprofilerate 1
+	go tool pprof bufs.test mem.out --alloc_space --nodefraction 0.0001 \
+	       --edgefraction 0	-web
+	@echo "Note: Foo vs FooBufs allocated memory is in hundreds of MBs vs 8 kB."

Godeps/_workspace/src/github.com/cznic/bufs/README.md

@@ -0,0 +1,8 @@
+bufs
+====
+
+Package bufs implements a simple buffer cache.
+
+ installation: go get github.com/cznic/bufs
+
+documentation: http://godoc.org/github.com/cznic/bufs

Godeps/_workspace/src/github.com/cznic/bufs/bufs.go

@@ -0,0 +1,391 @@
+// Copyright 2014 The bufs Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package bufs implements a simple buffer cache.
+//
+// The intended use scheme is like:
+//
+//	type Foo struct {
+//		buffers bufs.Buffers
+//		...
+//	}
+//
+//	// Bar can call Qux, but not the other way around (in this example).
+//	const maxFooDepth = 2
+//
+//	func NewFoo() *Foo {
+//		return &Foo{buffers: bufs.New(maxFooDepth), ...}
+//	}
+//
+//	func (f *Foo) Bar(n int) {
+//		buf := f.buffers.Alloc(n) // needed locally for computation and/or I/O
+//		defer f.buffers.Free()
+//		...
+//		f.Qux(whatever)
+//	}
+//
+//	func (f *Foo) Qux(n int) {
+//		buf := f.buffers.Alloc(n) // needed locally for computation and/or I/O
+//		defer f.buffers.Free()
+//		...
+//	}
+//
+// The whole idea behind 'bufs' is that when calling e.g. Foo.Bar N times, then
+// normally, without using 'bufs', there will be 2*N (in this example) []byte
+// buffers allocated.  While using 'bufs', only 2 buffers (in this example)
+// will ever be created. For large N it can be a substantial difference.
+//
+// It's not a good idea to use Buffers to cache too big buffers. The cost of
+// having a cached buffer is that the buffer is naturally not eligible for
+// garbage collection.  Of course, that holds only while the Foo instance is
+// reachable, in the above example.
+//
+// The buffer count limit is intentionally "hard" (read panicking), although
+// configurable in New().  The rationale is to prevent recursive calls, using
+// Alloc, to cause excessive, "static" memory consumption. Tune the limit
+// carefully or do not use Buffers from within [mutually] recursive functions
+// where the nesting depth is not realistically bounded to some rather small
+// number.
+//
+// Buffers cannot guarantee improvements to you program performance. There may
+// be a gain in case where they fit well. Firm grasp on what your code is
+// actually doing, when and in what order is essential to proper use of
+// Buffers. It's _highly_ recommended to first do profiling and memory
+// profiling before even thinking about using 'bufs'. The real world example,
+// and cause for this package, was a first correct, yet no optimizations done
+// version of a program; producing few MB of useful data while allocating 20+GB
+// of memory.  Of course the garbage collector properly kicked in, yet the
+// memory abuse caused ~80+% of run time to be spent memory management.  The
+// program _was_ expected to be slow in its still development phase, but the
+// bottleneck was guessed to be in I/O.  Actually the hard disk was waiting for
+// the billions bytes being allocated and zeroed. Garbage collect on low
+// memory, rinse and repeat.
+//
+// In the provided tests, TestFoo and TestFooBufs do the same simulated work,
+// except the later uses Buffers while the former does not. Suggested test runs
+// which show the differences:
+//
+//	$ go test -bench . -benchmem
+//
+//	or
+//
+//	$ go test -c
+//	$ ./bufs.test -test.v -test.run Foo -test.memprofile mem.out -test.memprofilerate 1
+//	$ go tool pprof bufs.test mem.out --alloc_space --nodefraction 0.0001 --edgefraction 0 -web
+//	$ # Note: Foo vs FooBufs allocated memory is in hundreds of MBs vs 8 kB.
+//
+//	or
+//
+//	$ make demo # same as all of the above
+//
+//
+// NOTE: Alloc/Free calls must be properly nested in the same way as in for
+// example BeginTransaction/EndTransaction pairs. If your code can panic then
+// the pairing should be enforced by deferred calls.
+//
+// NOTE: Buffers objects do not allocate any space until requested by Alloc,
+// the mechanism works on demand only.
+//
+// FAQ: Why the 'bufs' package name?
+//
+// Package name 'bufs' was intentionally chosen instead of the perhaps more
+// conventional 'buf'. There are already too many 'buf' named things in the
+// code out there and that'll be a source of a lot of trouble. It's a bit
+// similar situation as in the case of package "strings" (not "string").
+package bufs
+
+import (
+	"errors"
+	"sort"
+	"sync"
+)
+
+// Buffers type represents a buffer ([]byte) cache.
+//
+// NOTE: Do not modify Buffers directly, use only its methods. Do not create
+// additional values (copies) of Buffers, that'll break its functionality. Use
+// a pointer instead to refer to a single instance from different
+// places/scopes.
+type Buffers [][]byte
+
+// New returns a newly created instance of Buffers with a maximum capacity of n
+// buffers.
+//
+// NOTE: 'bufs.New(n)' is the same as 'make(bufs.Buffers, n)'.
+func New(n int) Buffers {
+	return make(Buffers, n)
+}
+
+// Alloc will return a buffer such that len(r) == n. It will firstly try to
+// find an existing and unused buffer of big enough size. Only when there is no
+// such, then one of the buffer slots is reallocated to a bigger size.
+//
+// It's okay to use append with buffers returned by Alloc. But it can cause
+// allocation in that case and will again be producing load for the garbage
+// collector. The best use of Alloc is for I/O buffers where the needed size of
+// the buffer is figured out at some point of the code path in a 'final size'
+// sense. Another real world example are compression/decompression buffers.
+//
+// NOTE: The buffer returned by Alloc _is not_ zeroed. That's okay for e.g.
+// passing a buffer to io.Reader. If you need a zeroed buffer use Calloc.
+//
+// NOTE: Buffers returned from Alloc _must not_ be exposed/returned to your
+// clients.  Those buffers are intended to be used strictly internally, within
+// the methods of some "object".
+//
+// NOTE: Alloc will panic if there are no buffers (buffer slots) left.
+func (p *Buffers) Alloc(n int) (r []byte) {
+	b := *p
+	if len(b) == 0 {
+		panic(errors.New("Buffers.Alloc: out of buffers"))
+	}
+
+	biggest, best, biggestI, bestI := -1, -1, -1, -1
+	for i, v := range b {
+		//ln := len(v)
+		// The above was correct, buts it's just confusing. It worked
+		// because not the buffers, but slices of them are returned in
+		// the 'if best >= n' code path.
+		ln := cap(v)
+
+		if ln >= biggest {
+			biggest, biggestI = ln, i
+		}
+
+		if ln >= n && (bestI < 0 || best > ln) {
+			best, bestI = ln, i
+			if ln == n {
+				break
+			}
+		}
+	}
+
+	last := len(b) - 1
+	if best >= n {
+		r = b[bestI]
+		b[last], b[bestI] = b[bestI], b[last]
+		*p = b[:last]
+		return r[:n]
+	}
+
+	r = make([]byte, n, overCommit(n))
+	b[biggestI] = r
+	b[last], b[biggestI] = b[biggestI], b[last]
+	*p = b[:last]
+	return
+}
+
+// Calloc will acquire a buffer using Alloc and then clears it to zeros. The
+// zeroing goes up to n, not cap(r).
+func (p *Buffers) Calloc(n int) (r []byte) {
+	r = p.Alloc(n)
+	for i := range r {
+		r[i] = 0
+	}
+	return
+}
+
+// Free makes the lastly allocated by Alloc buffer free (available) again for
+// Alloc.
+//
+// NOTE: Improper Free invocations, like in the sequence {New, Alloc, Free,
+// Free}, will panic.
+func (p *Buffers) Free() {
+	b := *p
+	b = b[:len(b)+1]
+	*p = b
+}
+
+// Stats reports memory consumed by Buffers, without accounting for some
+// (smallish) additional overhead.
+func (p *Buffers) Stats() (bytes int) {
+	b := *p
+	b = b[:cap(b)]
+	for _, v := range b {
+		bytes += cap(v)
+	}
+	return
+}
+
+// Cache caches buffers ([]byte). A zero value of Cache is ready for use.
+//
+// NOTE: Do not modify a Cache directly, use only its methods. Do not create
+// additional values (copies) of a Cache, that'll break its functionality. Use
+// a pointer instead to refer to a single instance from different
+// places/scopes.
+type Cache [][]byte
+
+// Get returns a buffer ([]byte) of length n. If no such buffer is cached then
+// a biggest cached buffer is resized to have length n and returned. If there
+// are no cached items at all, Get returns a newly allocated buffer.
+//
+// In other words the cache policy is:
+//
+// - If the cache is empty, the buffer must be newly created and returned.
+// Cache remains empty.
+//
+// - If a buffer of sufficient size is found in the cache, remove it from the
+// cache and return it.
+//
+// - Otherwise the cache is non empty, but no cached buffer is big enough.
+// Enlarge the biggest cached buffer, remove it from the cache and return it.
+// This provide cached buffers size adjustment based on demand.
+//
+// In short, if the cache is not empty, Get guarantees to make it always one
+// item less.  This rules prevent uncontrolled cache grow in some scenarios.
+// The older policy was not preventing that. Another advantage is better cached
+// buffers sizes "auto tuning", although not in every possible use case.
+//
+// NOTE: The buffer returned by Get _is not guaranteed_ to be zeroed. That's
+// okay for e.g.  passing a buffer to io.Reader. If you need a zeroed buffer
+// use Cget.
+func (c *Cache) Get(n int) []byte {
+	r, _ := c.get(n)
+	return r
+}
+
+func (c *Cache) get(n int) (r []byte, isZeroed bool) {
+	s := *c
+	lens := len(s)
+	if lens == 0 {
+		r, isZeroed = make([]byte, n, overCommit(n)), true
+		return
+	}
+
+	i := sort.Search(lens, func(x int) bool { return len(s[x]) >= n })
+	if i == lens {
+		i--
+		s[i] = make([]byte, n, overCommit(n))
+	}
+	r = s[i][:n]
+	copy(s[i:], s[i+1:])
+	s[lens-1] = nil
+	s = s[:lens-1]
+	*c = s
+	return r, false
+}
+
+// Cget will acquire a buffer using Get and then clears it to zeros. The
+// zeroing goes up to n, not cap(r).
+func (c *Cache) Cget(n int) (r []byte) {
+	r, ok := c.get(n)
+	if ok {
+		return
+	}
+
+	for i := range r {
+		r[i] = 0
+	}
+	return
+}
+
+// Put caches b for possible later reuse (via Get). No other references to b's
+// backing array may exist. Otherwise a big mess is sooner or later inevitable.
+func (c *Cache) Put(b []byte) {
+	b = b[:cap(b)]
+	lenb := len(b)
+	if lenb == 0 {
+		return
+	}
+
+	s := *c
+	lens := len(s)
+	i := sort.Search(lens, func(x int) bool { return len(s[x]) >= lenb })
+	s = append(s, nil)
+	copy(s[i+1:], s[i:])
+	s[i] = b
+	*c = s
+	return
+}
+
+// Stats reports memory consumed by a Cache, without accounting for some
+// (smallish) additional overhead. 'n' is the number of cached buffers, bytes
+// is their combined capacity.
+func (c Cache) Stats() (n, bytes int) {
+	n = len(c)
+	for _, v := range c {
+		bytes += cap(v)
+	}
+	return
+}
+
+// CCache is a Cache which is safe for concurrent use by multiple goroutines.
+type CCache struct {
+	c  Cache
+	mu sync.Mutex
+}
+
+// Get returns a buffer ([]byte) of length n. If no such buffer is cached then
+// a biggest cached buffer is resized to have length n and returned. If there
+// are no cached items at all, Get returns a newly allocated buffer.
+//
+// In other words the cache policy is:
+//
+// - If the cache is empty, the buffer must be newly created and returned.
+// Cache remains empty.
+//
+// - If a buffer of sufficient size is found in the cache, remove it from the
+// cache and return it.
+//
+// - Otherwise the cache is non empty, but no cached buffer is big enough.
+// Enlarge the biggest cached buffer, remove it from the cache and return it.
+// This provide cached buffers size adjustment based on demand.
+//
+// In short, if the cache is not empty, Get guarantees to make it always one
+// item less.  This rules prevent uncontrolled cache grow in some scenarios.
+// The older policy was not preventing that. Another advantage is better cached
+// buffers sizes "auto tuning", although not in every possible use case.
+//
+// NOTE: The buffer returned by Get _is not guaranteed_ to be zeroed. That's
+// okay for e.g.  passing a buffer to io.Reader. If you need a zeroed buffer
+// use Cget.
+func (c *CCache) Get(n int) []byte {
+	c.mu.Lock()
+	r, _ := c.c.get(n)
+	c.mu.Unlock()
+	return r
+}
+
+// Cget will acquire a buffer using Get and then clears it to zeros. The
+// zeroing goes up to n, not cap(r).
+func (c *CCache) Cget(n int) (r []byte) {
+	c.mu.Lock()
+	r = c.c.Cget(n)
+	c.mu.Unlock()
+	return
+}
+
+// Put caches b for possible later reuse (via Get). No other references to b's
+// backing array may exist. Otherwise a big mess is sooner or later inevitable.
+func (c *CCache) Put(b []byte) {
+	c.mu.Lock()
+	c.c.Put(b)
+	c.mu.Unlock()
+}
+
+// Stats reports memory consumed by a Cache, without accounting for some
+// (smallish) additional overhead. 'n' is the number of cached buffers, bytes
+// is their combined capacity.
+func (c *CCache) Stats() (n, bytes int) {
+	c.mu.Lock()
+	n, bytes = c.c.Stats()
+	c.mu.Unlock()
+	return
+}
+
+// GCache is a ready to use global instance of a CCache.
+var GCache CCache
+
+func overCommit(n int) int {
+	switch {
+	case n < 8:
+		return 8
+	case n < 1e5:
+		return 2 * n
+	case n < 1e6:
+		return 3 * n / 2
+	default:
+		return n
+	}
+}

Godeps/_workspace/src/github.com/cznic/bufs/bufs_test.go

@@ -0,0 +1,174 @@
+// Copyright 2014 The bufs Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package bufs
+
+import (
+	"fmt"
+	"path"
+	"runtime"
+	"testing"
+)
+
+var dbg = func(s string, va ...interface{}) {
+	_, fn, fl, _ := runtime.Caller(1)
+	fmt.Printf("%s:%d: ", path.Base(fn), fl)
+	fmt.Printf(s, va...)
+	fmt.Println()
+}
+
+func Test0(t *testing.T) {
+	b := New(0)
+	defer func() {
+		recover()
+	}()
+
+	b.Alloc(1)
+	t.Fatal("unexpected success")
+}
+
+func Test1(t *testing.T) {
+	b := New(1)
+	expected := false
+	defer func() {
+		if e := recover(); e != nil && !expected {
+			t.Fatal(fmt.Errorf("%v", e))
+		}
+	}()
+
+	b.Alloc(1)
+	expected = true
+	b.Alloc(1)
+	t.Fatal("unexpected success")
+}
+
+func Test2(t *testing.T) {
+	b := New(1)
+	expected := false
+	defer func() {
+		if e := recover(); e != nil && !expected {
+			t.Fatal(fmt.Errorf("%v", e))
+		}
+	}()
+
+	b.Alloc(1)
+	b.Free()
+	b.Alloc(1)
+	expected = true
+	b.Alloc(1)
+	t.Fatal("unexpected success")
+}
+
+func Test3(t *testing.T) {
+	b := New(1)
+	expected := false
+	defer func() {
+		if e := recover(); e != nil && !expected {
+			t.Fatal(fmt.Errorf("%v", e))
+		}
+	}()
+
+	b.Alloc(1)
+	b.Free()
+	expected = true
+	b.Free()
+	t.Fatal("unexpected success")
+}
+
+const (
+	N       = 1e5
+	bufSize = 1 << 12
+)
+
+type Foo struct {
+	result []byte
+}
+
+func NewFoo() *Foo {
+	return &Foo{}
+}
+
+func (f *Foo) Bar(n int) {
+	buf := make([]byte, n)
+	sum := 0
+	for _, v := range buf {
+		sum += int(v)
+	}
+	f.result = append(f.result, byte(sum))
+	f.Qux(n)
+}
+
+func (f *Foo) Qux(n int) {
+	buf := make([]byte, n)
+	sum := 0
+	for _, v := range buf {
+		sum += int(v)
+	}
+	f.result = append(f.result, byte(sum))
+}
+
+type FooBufs struct {
+	buffers Buffers
+	result  []byte
+}
+
+const maxFooDepth = 2
+
+func NewFooBufs() *FooBufs {
+	return &FooBufs{buffers: New(maxFooDepth)}
+}
+
+func (f *FooBufs) Bar(n int) {
+	buf := f.buffers.Alloc(n)
+	defer f.buffers.Free()
+
+	sum := 0
+	for _, v := range buf {
+		sum += int(v)
+	}
+	f.result = append(f.result, byte(sum))
+	f.Qux(n)
+}
+
+func (f *FooBufs) Qux(n int) {
+	buf := f.buffers.Alloc(n)
+	defer f.buffers.Free()
+
+	sum := 0
+	for _, v := range buf {
+		sum += int(v)
+	}
+	f.result = append(f.result, byte(sum))
+}
+
+func TestFoo(t *testing.T) {
+	foo := NewFoo()
+	for i := 0; i < N; i++ {
+		foo.Bar(bufSize)
+	}
+}
+
+func TestFooBufs(t *testing.T) {
+	foo := NewFooBufs()
+	for i := 0; i < N; i++ {
+		foo.Bar(bufSize)
+	}
+	t.Log("buffers.Stats()", foo.buffers.Stats())
+}
+
+func BenchmarkFoo(b *testing.B) {
+	b.SetBytes(2 * bufSize)
+	foo := NewFoo()
+	for i := 0; i < b.N; i++ {
+		foo.Bar(bufSize)
+	}
+}
+
+func BenchmarkFooBufs(b *testing.B) {
+	b.SetBytes(2 * bufSize)
+	foo := NewFooBufs()
+	for i := 0; i < b.N; i++ {
+		foo.Bar(bufSize)
+	}
+}

Godeps/_workspace/src/github.com/cznic/exp/lldb/2pc.go

@@ -0,0 +1,324 @@
+// Copyright 2014 The lldb Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Two Phase Commit & Structural ACID
+
+package lldb
+
+import (
+	"bufio"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/coreos/rocket/Godeps/_workspace/src/github.com/cznic/fileutil"
+	"github.com/coreos/rocket/Godeps/_workspace/src/github.com/cznic/mathutil"
+)
+
+var _ Filer = &ACIDFiler0{} // Ensure ACIDFiler0 is a Filer
+
+type acidWrite struct {
+	b   []byte
+	off int64
+}
+
+type acidWriter0 ACIDFiler0
+
+func (a *acidWriter0) WriteAt(b []byte, off int64) (n int, err error) {
+	f := (*ACIDFiler0)(a)
+	if f.bwal == nil { // new epoch
+		f.data = f.data[:0]
+		f.bwal = bufio.NewWriter(f.wal)
+		if err = a.writePacket([]interface{}{wpt00Header, walTypeACIDFiler0, ""}); err != nil {
+			return
+		}
+	}
+
+	if err = a.writePacket([]interface{}{wpt00WriteData, b, off}); err != nil {
+		return
+	}
+
+	f.data = append(f.data, acidWrite{b, off})
+	return len(b), nil
+}
+
+func (a *acidWriter0) writePacket(items []interface{}) (err error) {
+	f := (*ACIDFiler0)(a)
+	b, err := EncodeScalars(items...)
+	if err != nil {
+		return
+	}
+
+	var b4 [4]byte
+	binary.BigEndian.PutUint32(b4[:], uint32(len(b)))
+	if _, err = f.bwal.Write(b4[:]); err != nil {
+		return
+	}
+
+	if _, err = f.bwal.Write(b); err != nil {
+		return
+	}
+
+	if m := (4 + len(b)) % 16; m != 0 {
+		var pad [15]byte
+		_, err = f.bwal.Write(pad[:16-m])
+	}
+	return
+}
+
+// WAL Packet Tags
+const (
+	wpt00Header = iota
+	wpt00WriteData
+	wpt00Checkpoint
+)
+
+const (
+	walTypeACIDFiler0 = iota
+)
+
+// ACIDFiler0 is a very simple, synchronous implementation of 2PC. It uses a
+// single write ahead log file to provide the structural atomicity
+// (BeginUpdate/EndUpdate/Rollback) and durability (DB can be recovered from
+// WAL if a crash occurred).
+//
+// ACIDFiler0 is a Filer.
+//
+// NOTE: Durable synchronous 2PC involves three fsyncs in this implementation
+// (WAL, DB, zero truncated WAL).  Where possible, it's recommended to collect
+// transactions for, say one second before performing the two phase commit as
+// the typical performance for rotational hard disks is about few tens of
+// fsyncs per second atmost. For an example of such collective transaction
+// approach please see the colecting FSM STT in Dbm's documentation[1].
+//
+//  [1]: http://godoc.org/github.com/cznic/exp/dbm
+type ACIDFiler0 struct {
+	*RollbackFiler
+	wal               *os.File
+	bwal              *bufio.Writer
+	data              []acidWrite
+	testHook          bool  // keeps WAL untruncated (once)
+	peakWal           int64 // tracks WAL maximum used size
+	peakBitFilerPages int   // track maximum transaction memory
+}
+
+// NewACIDFiler0 returns a  newly created ACIDFiler0 with WAL in wal.
+//
+// If the WAL is zero sized then a previous clean shutdown of db is taken for
+// granted and no recovery procedure is taken.
+//
+// If the WAL is of non zero size then it is checked for having a
+// commited/fully finished transaction not yet been reflected in db. If such
+// transaction exists it's committed to db. If the recovery process finishes
+// successfully, the WAL is truncated to zero size and fsync'ed prior to return
+// from NewACIDFiler0.
+func NewACIDFiler(db Filer, wal *os.File) (r *ACIDFiler0, err error) {
+	fi, err := wal.Stat()
+	if err != nil {
+		return
+	}
+
+	r = &ACIDFiler0{wal: wal}
+
+	if fi.Size() != 0 {
+		if err = r.recoverDb(db); err != nil {
+			return
+		}
+	}
+
+	acidWriter := (*acidWriter0)(r)
+
+	if r.RollbackFiler, err = NewRollbackFiler(
+		db,
+		func(sz int64) (err error) {
+			// Checkpoint
+			if err = acidWriter.writePacket([]interface{}{wpt00Checkpoint, sz}); err != nil {
+				return
+			}
+
+			if err = r.bwal.Flush(); err != nil {
+				return
+			}
+
+			r.bwal = nil
+
+			if err = r.wal.Sync(); err != nil {
+				return
+			}
+
+			wfi, err := r.wal.Stat()
+			switch err != nil {
+			case true:
+				// unexpected, but ignored
+			case false:
+				r.peakWal = mathutil.MaxInt64(wfi.Size(), r.peakWal)
+			}
+
+			// Phase 1 commit complete
+
+			for _, v := range r.data {
+				if _, err := db.WriteAt(v.b, v.off); err != nil {
+					return err
+				}
+			}
+
+			if err = db.Truncate(sz); err != nil {
+				return
+			}
+
+			if err = db.Sync(); err != nil {
+				return
+			}
+
+			// Phase 2 commit complete
+
+			if !r.testHook {
+				if err = r.wal.Truncate(0); err != nil {
+					return
+				}
+
+				if _, err = r.wal.Seek(0, 0); err != nil {
+					return
+				}
+			}
+
+			r.testHook = false
+			return r.wal.Sync()
+
+		},
+		acidWriter,
+	); err != nil {
+		return
+	}
+
+	return r, nil
+}
+
+// PeakWALSize reports the maximum size WAL has ever used.
+func (a ACIDFiler0) PeakWALSize() int64 {
+	return a.peakWal
+}
+
+func (a *ACIDFiler0) readPacket(f *bufio.Reader) (items []interface{}, err error) {
+	var b4 [4]byte
+	n, err := io.ReadAtLeast(f, b4[:], 4)
+	if n != 4 {
+		return
+	}
+
+	ln := int(binary.BigEndian.Uint32(b4[:]))
+	m := (4 + ln) % 16
+	padd := (16 - m) % 16
+	b := make([]byte, ln+padd)
+	if n, err = io.ReadAtLeast(f, b, len(b)); n != len(b) {
+		return
+	}
+
+	return DecodeScalars(b[:ln])
+}
+
+func (a *ACIDFiler0) recoverDb(db Filer) (err error) {
+	fi, err := a.wal.Stat()
+	if err != nil {
+		return &ErrILSEQ{Type: ErrInvalidWAL, Name: a.wal.Name(), More: err}
+	}
+
+	if sz := fi.Size(); sz%16 != 0 {
+		return &ErrILSEQ{Type: ErrFileSize, Name: a.wal.Name(), Arg: sz}
+	}
+
+	f := bufio.NewReader(a.wal)
+	items, err := a.readPacket(f)
+	if err != nil {
+		return
+	}
+
+	if len(items) != 3 || items[0] != int64(wpt00Header) || items[1] != int64(walTypeACIDFiler0) {
+		return &ErrILSEQ{Type: ErrInvalidWAL, Name: a.wal.Name(), More: fmt.Sprintf("invalid packet items %#v", items)}
+	}
+
+	tr := NewBTree(nil)
+
+	for {
+		items, err = a.readPacket(f)
+		if err != nil {
+			return
+		}
+
+		if len(items) < 2 {
+			return &ErrILSEQ{Type: ErrInvalidWAL, Name: a.wal.Name(), More: fmt.Sprintf("too few packet items %#v", items)}
+		}
+
+		switch items[0] {
+		case int64(wpt00WriteData):
+			if len(items) != 3 {
+				return &ErrILSEQ{Type: ErrInvalidWAL, Name: a.wal.Name(), More: fmt.Sprintf("invalid data packet items %#v", items)}
+			}
+
+			b, off := items[1].([]byte), items[2].(int64)
+			var key [8]byte
+			binary.BigEndian.PutUint64(key[:], uint64(off))
+			if err = tr.Set(key[:], b); err != nil {
+				return
+			}
+		case int64(wpt00Checkpoint):
+			var b1 [1]byte
+			if n, err := f.Read(b1[:]); n != 0 || err == nil {
+				return &ErrILSEQ{Type: ErrInvalidWAL, Name: a.wal.Name(), More: fmt.Sprintf("checkpoint n %d, err %v", n, err)}
+			}
+
+			if len(items) != 2 {
+				return &ErrILSEQ{Type: ErrInvalidWAL, Name: a.wal.Name(), More: fmt.Sprintf("checkpoint packet invalid items %#v", items)}
+			}
+
+			sz := items[1].(int64)
+			enum, err := tr.seekFirst()
+			if err != nil {
+				return err
+			}
+
+			for {
+				k, v, err := enum.current()
+				if err != nil {
+					if fileutil.IsEOF(err) {
+						break
+					}
+
+					return err
+				}
+
+				if _, err = db.WriteAt(v, int64(binary.BigEndian.Uint64(k))); err != nil {
+					return err
+				}
+
+				if err = enum.next(); err != nil {
+					if fileutil.IsEOF(err) {
+						break
+					}
+
+					return err
+				}
+			}
+
+			if err = db.Truncate(sz); err != nil {
+				return err
+			}
+
+			if err = db.Sync(); err != nil {
+				return err
+			}
+
+			// Recovery complete
+
+			if err = a.wal.Truncate(0); err != nil {
+				return err
+			}
+
+			return a.wal.Sync()
+		default:
+			return &ErrILSEQ{Type: ErrInvalidWAL, Name: a.wal.Name(), More: fmt.Sprintf("packet tag %v", items[0])}
+		}
+	}
+}

Godeps/_workspace/src/github.com/cznic/exp/lldb/2pc_docs.go

@@ -0,0 +1,44 @@
+// Copyright 2014 The lldb Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+/*
+
+Anatomy of a WAL file
+
+WAL file
+	A sequence of packets
+
+WAL packet, parts in slice notation
+	[0:4],   4 bytes:        N uint32        // network byte order
+	[4:4+N], N bytes:        payload []byte  // gb encoded scalars
+
+Packets, including the 4 byte 'size' prefix, MUST BE padded to size == 0 (mod
+16). The values of the padding bytes MUST BE zero.
+
+Encoded scalars first item is a packet type number (packet tag). The meaning of
+any other item(s) of the payload depends on the packet tag.
+
+Packet definitions
+
+	{wpt00Header int, typ int, s string}
+		typ:	Must be zero (ACIDFiler0 file).
+		s:	Any comment string, empty string is okay.
+
+		This packet must be present only once - as the first packet of
+		a WAL file.
+
+	{wpt00WriteData int, b []byte, off int64}
+		Write data (WriteAt(b, off)).
+
+	{wpt00Checkpoint int, sz int64}
+		Checkpoint (Truncate(sz)).
+
+		This packet must be present only once - as the last packet of
+		a WAL file.
+
+*/
+
+package lldb
+
+//TODO optimize bitfiler/wal/2pc data above final size

Godeps/_workspace/src/github.com/cznic/exp/lldb/2pc_test.go

@@ -0,0 +1,285 @@
+// Copyright 2014 The lldb Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Two Phase Commit & Structural ACID
+
+package lldb
+
+import (
+	"bytes"
+	"encoding/binary"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"testing"
+
+	"github.com/coreos/rocket/Godeps/_workspace/src/github.com/cznic/mathutil"
+)
+
+var _ Filer = &truncFiler{}
+
+type truncFiler struct {
+	f            Filer
+	fake         *MemFiler
+	totalWritten int // Including silently dropped
+	realWritten  int
+	limit        int // -1: unlimited, n: silently stop writing after limit bytes
+}
+
+func NewTruncFiler(f Filer, limit int) *truncFiler {
+	return &truncFiler{f: f, fake: NewMemFiler(), limit: limit}
+}
+
+func (f *truncFiler) BeginUpdate() error                      { panic("internal error") }
+func (f *truncFiler) Close() error                            { return f.f.Close() }
+func (f *truncFiler) EndUpdate() error                        { panic("internal error") }
+func (f *truncFiler) Name() string                            { return f.f.Name() }
+func (f *truncFiler) PunchHole(off, sz int64) error           { panic("internal error") }
+func (f *truncFiler) ReadAt(b []byte, off int64) (int, error) { return f.fake.ReadAt(b, off) }
+func (f *truncFiler) Rollback() error                         { panic("internal error") }
+func (f *truncFiler) Size() (int64, error)                    { return f.fake.Size() }
+func (f *truncFiler) Sync() error                             { return f.f.Sync() }
+
+func (f *truncFiler) Truncate(sz int64) error {
+	f.fake.Truncate(sz)
+	return f.f.Truncate(sz)
+}
+
+func (f *truncFiler) WriteAt(b []byte, off int64) (n int, err error) {
+	rq := len(b)
+	n = f.totalWritten
+	if lim := f.limit; lim >= 0 && n+rq > lim {
+		over := n + rq - lim
+		rq -= over
+		rq = mathutil.Max(rq, 0)
+	}
+
+	if n, err = f.fake.WriteAt(b, off); err != nil {
+		return
+	}
+
+	f.totalWritten += n
+	if rq != 0 {
+		n, err := f.f.WriteAt(b[:rq], off)
+		if err != nil {
+			return n, err
+		}
+		f.realWritten += n
+	}
+	return
+}
+
+// Verify memory BTrees don't have maxRq limits.
+func TestACID0MemBTreeCaps(t *testing.T) {
+	rng := rand.New(rand.NewSource(42))
+	tr := NewBTree(nil)
+	b := make([]byte, 2*maxRq)
+	for i := range b {
+		b[i] = byte(rng.Int())
+	}
+
+	if err := tr.Set(nil, b); err != nil {
+		t.Fatal(len(b), err)
+	}
+
+	g, err := tr.Get(nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !bytes.Equal(g, b) {
+		t.Fatal("data mismatach")
+	}
+}
+
+func TestACIDFiler0(t *testing.T) {
+	const SZ = 1 << 17
+
+	// Phase 1: Create a DB, fill with it with data.
+
+	wal, err := ioutil.TempFile("", "test-acidfiler0-wal-")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !*oKeep {
+		defer os.Remove(wal.Name())
+	}
+
+	db, err := ioutil.TempFile("", "test-acidfiler0-db-")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	dbName := db.Name()
+	if !*oKeep {
+		defer os.Remove(db.Name())
+	}
+
+	realFiler := NewSimpleFileFiler(db)
+	truncFiler := NewTruncFiler(realFiler, -1)
+	acidFiler, err := NewACIDFiler(truncFiler, wal)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	if err = acidFiler.BeginUpdate(); err != nil {
+		t.Error(err)
+		return
+	}
+
+	a, err := NewAllocator(acidFiler, &Options{})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	a.Compress = true
+
+	tr, h, err := CreateBTree(a, nil)
+	if h != 1 || err != nil {
+		t.Error(h, err)
+		return
+	}
+
+	rng := rand.New(rand.NewSource(42))
+	var key, val [8]byte
+	ref := map[int64]int64{}
+
+	for {
+		sz, err := acidFiler.Size()
+		if err != nil {
+			t.Error(err)
+			return
+		}
+
+		if sz > SZ {
+			break
+		}
+
+		k, v := rng.Int63(), rng.Int63()
+		ref[k] = v
+		binary.BigEndian.PutUint64(key[:], uint64(k))
+		binary.BigEndian.PutUint64(val[:], uint64(v))
+		if err := tr.Set(key[:], val[:]); err != nil {
+			t.Error(err)
+			return
+		}
+	}
+
+	acidFiler.testHook = true // keep WAL
+
+	if err := acidFiler.EndUpdate(); err != nil {
+		t.Error(err)
+		return
+	}
+
+	if err := acidFiler.Close(); err != nil {
+		t.Error(err)
+		return
+	}
+
+	if err := wal.Sync(); err != nil {
+		t.Error(err)
+		return
+	}
+
+	if _, err = wal.Seek(0, 0); err != nil {
+		t.Error(err)
+		return
+	}
+
+	// Phase 2: Reopen and verify structure and data.
+	db, err = os.OpenFile(dbName, os.O_RDWR, 0666)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	filer := NewSimpleFileFiler(db)
+	a, err = NewAllocator(filer, &Options{})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	if err = a.Verify(NewMemFiler(), nil, nil); err != nil {
+		t.Error(err)
+		return
+	}
+
+	tr, err = OpenBTree(a, nil, 1)
+	for k, v := range ref {
+		binary.BigEndian.PutUint64(key[:], uint64(k))
+		binary.BigEndian.PutUint64(val[:], uint64(v))
+		var b []byte
+		b, err = tr.Get(b, key[:])
+		if err != nil || b == nil || !bytes.Equal(b, val[:]) {
+			t.Error(err, b, val[:])
+			return
+		}
+	}
+
+	okImage, err := ioutil.ReadFile(dbName)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	// Phase 3: Simulate a crash
+	sz, err := filer.Size()
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	sz /= 2
+	if err := db.Truncate(sz); err != nil {
+		t.Error(err)
+		return
+	}
+
+	z := make([]byte, sz/3)
+	n, err := db.WriteAt(z, sz/3)
+	if n != len(z) {
+		t.Error(n, err)
+		return
+	}
+
+	if err := db.Sync(); err != nil {
+		t.Error(err)
+		return
+	}
+
+	// Phase 4: Open the corrupted DB
+	filer = NewSimpleFileFiler(db)
+	acidFiler, err = NewACIDFiler(filer, wal)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	if err = acidFiler.Sync(); err != nil {
+		t.Error(err)
+		return
+	}
+
+	if err = acidFiler.Close(); err != nil {
+		t.Error(err)
+		return
+	}
+
+	// Phase 5: Verify DB was recovered.
+	newImage, err := ioutil.ReadFile(dbName)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	if !bytes.Equal(okImage, newImage) {
+		t.Error(err)
+		return
+	}
+}

Godeps/_workspace/src/github.com/cznic/exp/lldb/AUTHORS

@@ -0,0 +1,11 @@
+# This file lists authors for copyright purposes.  This file is distinct from
+# the CONTRIBUTORS files.  See the latter for an explanation.
+#
+# Names should be added to this file as:
+#     Name or Organization <email address>
+#
+# The email address is not required for organizations.
+#
+# Please keep the list sorted.
+
+Jan Mercl <0xjnml@gmail.com>

Godeps/_workspace/src/github.com/cznic/exp/lldb/CONTRIBUTORS

@@ -0,0 +1,9 @@
+# This file lists people who contributed code to this repository.  The AUTHORS
+# file lists the copyright holders; this file lists people.
+#
+# Names should be added to this file like so:
+#     Name <email address>
+#
+# Please keep the list sorted.
+
+Jan Mercl <0xjnml@gmail.com>

Godeps/_workspace/src/github.com/cznic/exp/lldb/LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2014 The lldb Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the names of the authors nor the names of the
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

Godeps/_workspace/src/github.com/cznic/exp/lldb/Makefile

@@ -0,0 +1,45 @@
+# Copyright 2014 The lldb Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+.PHONY: all editor clean cover nuke
+
+testbin=lldb.test
+grep=--include=*.go
+
+all: editor
+	go build
+	go vet
+	golint .
+	go install
+	make todo
+
+clean:
+	go clean
+	rm -f *~ cov cov.html bad-dump good-dump lldb.test old.txt new.txt \
+		test-acidfiler0-* _test.db _wal
+
+cover:
+	t=$(shell tempfile) ; go test -coverprofile $$t && go tool cover -html $$t && unlink $$t
+
+editor:
+	go fmt
+	go test -i
+	go test -timeout 1h
+	
+mem:
+	go test -c
+	./$(testbin) -test.bench . -test.memprofile mem.out -test.memprofilerate 1 -test.timeout 24h
+	go tool pprof --lines --web --alloc_space $(testbin) mem.out
+
+nuke: clean
+	go clean -i
+
+todo:
+	@grep -nr $(grep) BUG * || true
+	@grep -nr $(grep) LATER * || true
+	@grep -nr $(grep) MAYBE * || true
+	@grep -nr $(grep) TODO * || true
+	@grep -nr $(grep) FIXME * || true
+	@grep -nr $(grep) ^[[:space:]]*_[[:space:]]*=[[:space:]][[:alpha:]][[:alnum:]]* * || true
+	@grep -nr $(grep) println * || true

Godeps/_workspace/src/github.com/cznic/exp/lldb/README.md

@@ -0,0 +1,8 @@
+lldb
+====
+
+Package lldb (WIP) implements a low level database engine.
+
+Installation: $ go get github.com/cznic/exp/lldb
+
+Documentation: [godoc.org/github.com/cznic/exp/lldb](http://godoc.org/github.com/cznic/exp/lldb)

Godeps/_workspace/src/github.com/cznic/exp/lldb/all_test.go

@@ -0,0 +1,43 @@
+// Copyright 2014 The lldb Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package lldb
+
+import (
+	"encoding/hex"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"runtime"
+	"time"
+)
+
+const (
+	testDbName = "_test.db"
+	walName    = "_wal"
+)
+
+func now() time.Time { return time.Now() }
+
+func hdump(b []byte) string {
+	return hex.Dump(b)
+}
+
+func die() {
+	os.Exit(1)
+}
+
+func stack() string {
+	buf := make([]byte, 1<<16)
+	return string(buf[:runtime.Stack(buf, false)])
+}
+
+func temp() (dir, name string) {
+	dir, err := ioutil.TempDir("", "test-lldb-")
+	if err != nil {
+		panic(err)
+	}
+
+	return dir, filepath.Join(dir, "test.tmp")
+}

Godeps/_workspace/src/github.com/cznic/exp/lldb/errors.go

@@ -0,0 +1,170 @@
+// Copyright 2014 The lldb Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Some errors returned by this package.
+//
+// Note that this package can return more errors than declared here, for
+// example io.EOF from Filer.ReadAt().
+
+package lldb
+
+import (
+	"fmt"
+)
+
+// ErrDecodeScalars is possibly returned from DecodeScalars
+type ErrDecodeScalars struct {
+	B []byte // Data being decoded
+	I int    // offending offset
+}
+
+// Error implements the built in error type.
+func (e *ErrDecodeScalars) Error() string {
+	return fmt.Sprintf("DecodeScalars: corrupted data @ %d/%d", e.I, len(e.B))
+}
+
+// ErrINVAL reports invalid values passed as parameters, for example negative
+// offsets where only non-negative ones are allowed or read from the DB.
+type ErrINVAL struct {
+	Src string
+	Val interface{}
+}
+
+// Error implements the built in error type.
+func (e *ErrINVAL) Error() string {
+	return fmt.Sprintf("%s: %+v", e.Src, e.Val)
+}
+
+// ErrPERM is for example reported when a Filer is closed while BeginUpdate(s)
+// are not balanced with EndUpdate(s)/Rollback(s) or when EndUpdate or Rollback
+// is invoked which is not paired with a BeginUpdate.
+type ErrPERM struct {
+	Src string
+}
+
+// Error implements the built in error type.
+func (e *ErrPERM) Error() string {
+	return fmt.Sprintf("%s: Operation not permitted", string(e.Src))
+}
+
+// ErrTag represents an ErrILSEQ kind.
+type ErrType int
+
+// ErrILSEQ types
+const (
+	ErrOther ErrType = iota
+
+	ErrAdjacentFree          // Adjacent free blocks (.Off and .Arg)
+	ErrDecompress            // Used compressed block: corrupted compression
+	ErrExpFreeTag            // Expected a free block tag, got .Arg
+	ErrExpUsedTag            // Expected a used block tag, got .Arg
+	ErrFLT                   // Free block is invalid or referenced multiple times
+	ErrFLTLoad               // FLT truncated to .Off, need size >= .Arg
+	ErrFLTSize               // Free block size (.Arg) doesn't belong to its list min size: .Arg2
+	ErrFileSize              // File .Name size (.Arg) != 0 (mod 16)
+	ErrFreeChaining          // Free block, .prev.next doesn't point back to this block
+	ErrFreeTailBlock         // Last block is free
+	ErrHead                  // Head of a free block list has non zero Prev (.Arg)
+	ErrInvalidRelocTarget    // Reloc doesn't target (.Arg) a short or long used block
+	ErrInvalidWAL            // Corrupted write ahead log. .Name: file name, .More: more
+	ErrLongFreeBlkTooLong    // Long free block spans beyond EOF, size .Arg
+	ErrLongFreeBlkTooShort   // Long free block must have at least 2 atoms, got only .Arg
+	ErrLongFreeNextBeyondEOF // Long free block .Next (.Arg) spans beyond EOF
+	ErrLongFreePrevBeyondEOF // Long free block .Prev (.Arg) spans beyond EOF
+	ErrLongFreeTailTag       // Expected a long free block tail tag, got .Arg
+	ErrLostFreeBlock         // Free block is not in any FLT list
+	ErrNullReloc             // Used reloc block with nil target
+	ErrRelocBeyondEOF        // Used reloc points (.Arg) beyond EOF
+	ErrShortFreeTailTag      // Expected a short free block tail tag, got .Arg
+	ErrSmall                 // Request for a free block (.Arg) returned a too small one (.Arg2) at .Off
+	ErrTailTag               // Block at .Off has invalid tail CC (compression code) tag, got .Arg
+	ErrUnexpReloc            // Unexpected reloc block referred to from reloc block .Arg
+	ErrVerifyPadding         // Used block has nonzero padding
+	ErrVerifyTailSize        // Long free block size .Arg but tail size .Arg2
+	ErrVerifyUsedSpan        // Used block size (.Arg) spans beyond EOF
+)
+
+// ErrILSEQ reports a corrupted file format. Details in fields according to Type.
+type ErrILSEQ struct {
+	Type ErrType
+	Off  int64
+	Arg  int64
+	Arg2 int64
+	Arg3 int64
+	Name string
+	More interface{}
+}
+
+// Error implements the built in error type.
+func (e *ErrILSEQ) Error() string {
+	switch e.Type {
+	case ErrAdjacentFree:
+		return fmt.Sprintf("Adjacent free blocks at offset %#x and %#x", e.Off, e.Arg)
+	case ErrDecompress:
+		return fmt.Sprintf("Compressed block at offset %#x: Corrupted compressed content", e.Off)
+	case ErrExpFreeTag:
+		return fmt.Sprintf("Block at offset %#x: Expected a free block tag, got %#2x", e.Off, e.Arg)
+	case ErrExpUsedTag:
+		return fmt.Sprintf("Block at ofset %#x: Expected a used block tag, got %#2x", e.Off, e.Arg)
+	case ErrFLT:
+		return fmt.Sprintf("Free block at offset %#x is invalid or referenced multiple times", e.Off)
+	case ErrFLTLoad:
+		return fmt.Sprintf("FLT truncated to size %d, expected at least %d", e.Off, e.Arg)
+	case ErrFLTSize:
+		return fmt.Sprintf("Free block at offset %#x has size (%#x) should be at least (%#x)", e.Off, e.Arg, e.Arg2)
+	case ErrFileSize:
+		return fmt.Sprintf("File %q size (%#x) != 0 (mod 16)", e.Name, e.Arg)
+	case ErrFreeChaining:
+		return fmt.Sprintf("Free block at offset %#x: .prev.next doesn point back here.", e.Off)
+	case ErrFreeTailBlock:
+		return fmt.Sprintf("Free block at offset %#x: Cannot be last file block", e.Off)
+	case ErrHead:
+		return fmt.Sprintf("Block at offset %#x: Head of free block list has non zero .prev %#x", e.Off, e.Arg)
+	case ErrInvalidRelocTarget:
+		return fmt.Sprintf("Used reloc block at offset %#x: Target (%#x) is not a short or long used block", e.Off, e.Arg)
+	case ErrInvalidWAL:
+		return fmt.Sprintf("Corrupted write ahead log file: %q %v", e.Name, e.More)
+	case ErrLongFreeBlkTooLong:
+		return fmt.Sprintf("Long free block at offset %#x: Size (%#x) beyond EOF", e.Off, e.Arg)
+	case ErrLongFreeBlkTooShort:
+		return fmt.Sprintf("Long free block at offset %#x: Size (%#x) too small", e.Off, e.Arg)
+	case ErrLongFreeNextBeyondEOF:
+		return fmt.Sprintf("Long free block at offset %#x: Next (%#x) points beyond EOF", e.Off, e.Arg)
+	case ErrLongFreePrevBeyondEOF:
+		return fmt.Sprintf("Long free block at offset %#x: Prev (%#x) points beyond EOF", e.Off, e.Arg)
+	case ErrLongFreeTailTag:
+		return fmt.Sprintf("Block at offset %#x: Expected long free tail tag, got %#2x", e.Off, e.Arg)
+	case ErrLostFreeBlock:
+		return fmt.Sprintf("Free block at offset %#x: not in any FLT list", e.Off)
+	case ErrNullReloc:
+		return fmt.Sprintf("Used reloc block at offset %#x: Nil target", e.Off)
+	case ErrRelocBeyondEOF:
+		return fmt.Sprintf("Used reloc block at offset %#x: Link (%#x) points beyond EOF", e.Off, e.Arg)
+	case ErrShortFreeTailTag:
+		return fmt.Sprintf("Block at offset %#x: Expected short free tail tag, got %#2x", e.Off, e.Arg)
+	case ErrSmall:
+		return fmt.Sprintf("Request for of free block of size %d returned a too small (%d) one at offset %#x", e.Arg, e.Arg2, e.Off)
+	case ErrTailTag:
+		return fmt.Sprintf("Block at offset %#x: Invalid tail CC tag, got %#2x", e.Off, e.Arg)
+	case ErrUnexpReloc:
+		return fmt.Sprintf("Block at offset %#x: Unexpected reloc block. Referred to from reloc block at offset %#x", e.Off, e.Arg)
+	case ErrVerifyPadding:
+		return fmt.Sprintf("Used block at offset %#x: Nonzero padding", e.Off)
+	case ErrVerifyTailSize:
+		return fmt.Sprintf("Long free block at offset %#x: Size %#x, but tail size %#x", e.Off, e.Arg, e.Arg2)
+	case ErrVerifyUsedSpan:
+		return fmt.Sprintf("Used block at offset %#x: Size %#x spans beyond EOF", e.Off, e.Arg)
+	}
+
+	more := ""
+	if e.More != nil {
+		more = fmt.Sprintf(", %v", e.More)
+	}
+	off := ""
+	if e.Off != 0 {
+		off = fmt.Sprintf(", off: %#x", e.Off)
+	}
+
+	return fmt.Sprintf("Error%s%s", off, more)
+}

Godeps/_workspace/src/github.com/cznic/exp/lldb/filer.go

@@ -0,0 +1,192 @@
+// Copyright 2014 The lldb Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// An abstraction of file like (persistent) storage with optional (abstracted)
+// support for structural integrity.
+
+package lldb
+
+import (
+	"fmt"
+
+	"github.com/coreos/rocket/Godeps/_workspace/src/github.com/cznic/mathutil"
+)
+
+func doubleTrouble(first, second error) error {
+	return fmt.Errorf("%q. Additionally, while attempting to recover (rollback): %q", first, second)
+}
+
+// A Filer is a []byte-like model of a file or similar entity. It may
+// optionally implement support for structural transaction safety. In contrast
+// to a file stream, a Filer is not sequentially accessible. ReadAt and WriteAt
+// are always "addressed" by an offset and are assumed to perform atomically.
+// A Filer is not safe for concurrent access, it's designed for consumption by
+// the other objects in package, which should use a Filer from one goroutine
+// only or via a mutex. BeginUpdate, EndUpdate and Rollback must be either all
+// implemented by a Filer for structural integrity - or they should be all
+// no-ops; where/if that requirement is relaxed.
+//
+// If a Filer wraps another Filer implementation, it usually invokes the same
+// methods on the "inner" one, after some possible argument translations etc.
+// If a Filer implements the structural transactions handling methods
+// (BeginUpdate, EndUpdate and Rollback) as no-ops _and_ wraps another Filer:
+// it then still MUST invoke those methods on the inner Filer. This is
+// important for the case where a RollbackFiler exists somewhere down the
+// chain.  It's also important for an Allocator - to know when it must
+// invalidate its FLT cache.
+type Filer interface {
+	// BeginUpdate increments the "nesting" counter (initially zero). Every
+	// call to BeginUpdate must be eventually "balanced" by exactly one of
+	// EndUpdate or Rollback. Calls to BeginUpdate may nest.
+	BeginUpdate() error
+
+	// Analogous to os.File.Close().
+	Close() error
+
+	// EndUpdate decrements the "nesting" counter. If it's zero after that
+	// then assume the "storage" has reached structural integrity (after a
+	// batch of partial updates). If a Filer implements some support for
+	// that (write ahead log, journal, etc.) then the appropriate actions
+	// are to be taken for nesting == 0. Invocation of an unbalanced
+	// EndUpdate is an error.
+	EndUpdate() error
+
+	// Analogous to os.File.Name().
+	Name() string
+
+	// PunchHole deallocates space inside a "file" in the byte range
+	// starting at off and continuing for size bytes. The actual hole
+	// created by PunchHole may be smaller than requested. The Filer size
+	// (as reported by `Size()` does not change when hole punching, even
+	// when punching the end of a file off.  In contrast to the Linux
+	// implementation of FALLOC_FL_PUNCH_HOLE in `fallocate`(2); a Filer is
+	// free not only to ignore `PunchHole()` (implement it as a nop), but
+	// additionally no guarantees about the content of the hole, when
+	// eventually read back, are required, i.e.  any data, not only zeros,
+	// can be read from the "hole", including just anything what was left
+	// there - with all of the possible security problems.
+	PunchHole(off, size int64) error
+
+	// As os.File.ReadAt. Note: `off` is an absolute "file pointer"
+	// address and cannot be negative even when a Filer is a InnerFiler.
+	ReadAt(b []byte, off int64) (n int, err error)
+
+	// Rollback cancels and undoes the innermost pending update level.
+	// Rollback decrements the "nesting" counter.  If a Filer implements
+	// some support for keeping structural integrity (write ahead log,
+	// journal, etc.) then the appropriate actions are to be taken.
+	// Invocation of an unbalanced Rollback is an error.
+	Rollback() error
+
+	// Analogous to os.File.FileInfo().Size().
+	Size() (int64, error)
+
+	// Analogous to os.Sync().
+	Sync() (err error)
+
+	// Analogous to os.File.Truncate().
+	Truncate(size int64) error
+
+	// Analogous to os.File.WriteAt(). Note: `off` is an absolute "file
+	// pointer" address and cannot be negative even when a Filer is a
+	// InnerFiler.
+	WriteAt(b []byte, off int64) (n int, err error)
+}
+
+var _ Filer = &InnerFiler{} // Ensure InnerFiler is a Filer.
+
+// A InnerFiler is a Filer with added addressing/size translation.
+type InnerFiler struct {
+	outer Filer
+	off   int64
+}
+
+// NewInnerFiler returns a new InnerFiler wrapped by `outer` in a way which
+// adds `off` to every access.
+//
+// For example, considering:
+//
+// 	inner := NewInnerFiler(outer, 10)
+//
+// then
+//
+// 	inner.WriteAt([]byte{42}, 4)
+//
+// translates to
+//
+// 	outer.WriteAt([]byte{42}, 14)
+//
+// But an attempt to emulate
+//
+// 	outer.WriteAt([]byte{17}, 9)
+//
+// by
+//
+// 	inner.WriteAt([]byte{17}, -1)
+//
+// will fail as the `off` parameter can never be < 0. Also note that
+//
+// 	inner.Size() == outer.Size() - off,
+//
+// i.e. `inner` pretends no `outer` exists. Finally, after e.g.
+//
+// 	inner.Truncate(7)
+// 	outer.Size() == 17
+//
+// will be true.
+func NewInnerFiler(outer Filer, off int64) *InnerFiler { return &InnerFiler{outer, off} }
+
+// BeginUpdate implements Filer.
+func (f *InnerFiler) BeginUpdate() error { return f.outer.BeginUpdate() }
+
+// Close implements Filer.
+func (f *InnerFiler) Close() (err error) { return f.outer.Close() }
+
+// EndUpdate implements Filer.
+func (f *InnerFiler) EndUpdate() error { return f.outer.EndUpdate() }
+
+// Name implements Filer.
+func (f *InnerFiler) Name() string { return f.outer.Name() }
+
+// PunchHole implements Filer. `off`, `size` must be >= 0.
+func (f *InnerFiler) PunchHole(off, size int64) error { return f.outer.PunchHole(f.off+off, size) }
+
+// ReadAt implements Filer. `off` must be >= 0.
+func (f *InnerFiler) ReadAt(b []byte, off int64) (n int, err error) {
+	if off < 0 {
+		return 0, &ErrINVAL{f.outer.Name() + ":ReadAt invalid off", off}
+	}
+
+	return f.outer.ReadAt(b, f.off+off)
+}
+
+// Rollback implements Filer.
+func (f *InnerFiler) Rollback() error { return f.outer.Rollback() }
+
+// Size implements Filer.
+func (f *InnerFiler) Size() (int64, error) {
+	sz, err := f.outer.Size()
+	if err != nil {
+		return 0, err
+	}
+
+	return mathutil.MaxInt64(sz-f.off, 0), nil
+}
+
+// Sync() implements Filer.
+func (f *InnerFiler) Sync() (err error) {
+	return f.outer.Sync()
+}
+
+// Truncate implements Filer.
+func (f *InnerFiler) Truncate(size int64) error { return f.outer.Truncate(size + f.off) }
+
+// WriteAt implements Filer. `off` must be >= 0.
+func (f *InnerFiler) WriteAt(b []byte, off int64) (n int, err error) {
+	if off < 0 {
+		return 0, &ErrINVAL{f.outer.Name() + ":WriteAt invalid off", off}
+	}
+
+	return f.outer.WriteAt(b, f.off+off)
+}

Godeps/_workspace/src/github.com/cznic/exp/lldb/filer_test.go

@@ -0,0 +1,764 @@
+// Copyright 2014 The lldb Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package lldb
+
+import (
+	"bytes"
+	"encoding/hex"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"runtime"
+	"testing"
+
+	"github.com/coreos/rocket/Godeps/_workspace/src/github.com/cznic/fileutil"
+)
+
+// Bench knobs.
+const (
+	filerTestChunkSize = 32e3
+	filerTotalSize     = 10e6
+)
+
+type newFunc func() Filer
+
+type testFileFiler struct {
+	Filer
+}
+
+func (t *testFileFiler) Close() (err error) {
+	n := t.Name()
+	err = t.Filer.Close()
+	if errDel := os.Remove(n); errDel != nil && err == nil {
+		err = errDel
+	}
+	return
+}
+
+var (
+	newFileFiler = func() Filer {
+		file, err := ioutil.TempFile("", "lldb-test-file")
+		if err != nil {
+			panic(err)
+		}
+
+		return &testFileFiler{NewSimpleFileFiler(file)}
+	}
+
+	newOSFileFiler = func() Filer {
+		file, err := ioutil.TempFile("", "lldb-test-osfile")
+		if err != nil {
+			panic(err)
+		}
+
+		return &testFileFiler{NewOSFiler(file)}
+	}
+
+	newMemFiler = func() Filer {
+		return NewMemFiler()
+	}
+
+	nwBitFiler = func() Filer {
+		f, err := newBitFiler(NewMemFiler())
+		if err != nil {
+			panic(err)
+		}
+
+		return f
+	}
+
+	newRollbackFiler = func() Filer {
+		f := NewMemFiler()
+
+		var r Filer
+
+		checkpoint := func(sz int64) (err error) {
+			return f.Truncate(sz)
+		}
+
+		r, err := NewRollbackFiler(f, checkpoint, f)
+		if err != nil {
+			panic(err)
+		}
+
+		return r
+	}
+)
+
+func TestFilerNesting(t *testing.T) {
+	testFilerNesting(t, newFileFiler)
+	testFilerNesting(t, newOSFileFiler)
+	testFilerNesting(t, newMemFiler)
+	testFilerNesting(t, newRollbackFiler)
+}
+
+func testFilerNesting(t *testing.T, nf newFunc) {
+	// Check {Create, Close} works.
+	f := nf()
+	t.Log(f.Name())
+	if err := f.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	// Check {Create, EndUpdate} doesn't work.
+	f = nf()
+	t.Log(f.Name())
+	if err := f.EndUpdate(); err == nil {
+		f.Close()
+		t.Fatal("unexpected success")
+	}
+
+	if err := f.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	// Check {Create, BeginUpdate, Close} doesn't work.
+	f = nf()
+	t.Log(f.Name())
+	f.BeginUpdate()
+
+	if err := f.Close(); err == nil {
+		t.Fatal("unexpected success")
+	}
+
+	// Check {Create, BeginUpdate, EndUpdate, Close} works.
+	f = nf()
+	t.Log(f.Name())
+	f.BeginUpdate()
+	if err := f.EndUpdate(); err != nil {
+		f.Close()
+		t.Fatal(err)
+	}
+
+	if err := f.Close(); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestFilerTruncate(t *testing.T) {
+	testFilerTruncate(t, newFileFiler)
+	testFilerTruncate(t, newOSFileFiler)
+	testFilerTruncate(t, newMemFiler)
+	testFilerTruncate(t, nwBitFiler)
+	testFilerTruncate(t, newRollbackFiler)
+}
+
+func testFilerTruncate(t *testing.T, nf newFunc) {
+	f := nf()
+	t.Log(f.Name())
+	defer func() {
+		if err := f.Close(); err != nil {
+			t.Error(err)
+		}
+	}()
+
+	if _, ok := f.(*RollbackFiler); ok {
+		if err := f.BeginUpdate(); err != nil {
+			t.Fatal(err)
+		}
+
+		defer func() {
+			if err := f.EndUpdate(); err != nil {
+				t.Error(err)
+			}
+		}()
+	}
+
+	// Check Truncate works.
+	sz := int64(1e6)
+	if err := f.Truncate(sz); err != nil {
+		t.Error(err)
+		return
+	}
+
+	fsz, err := f.Size()
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	if g, e := fsz, sz; g != e {
+		t.Error(g, e)
+		return
+	}
+
+	sz *= 2
+	if err := f.Truncate(sz); err != nil {
+		t.Error(err)
+		return
+	}
+
+	fsz, err = f.Size()
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	if g, e := fsz, sz; g != e {
+		t.Error(g, e)
+		return
+	}
+
+	sz = 0
+	if err := f.Truncate(sz); err != nil {
+		t.Error(err)
+		return
+	}
+
+	fsz, err = f.Size()
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	if g, e := fsz, sz; g != e {
+		t.Error(g, e)
+		return
+	}
+
+	// Check Truncate(-1) doesn't work.
+	sz = -1
+	if err := f.Truncate(sz); err == nil {
+		t.Error(err)
+		return
+	}
+
+}
+
+func TestFilerReadAtWriteAt(t *testing.T) {
+	testFilerReadAtWriteAt(t, newFileFiler)
+	testFilerReadAtWriteAt(t, newOSFileFiler)
+	testFilerReadAtWriteAt(t, newMemFiler)
+	testFilerReadAtWriteAt(t, nwBitFiler)
+	testFilerReadAtWriteAt(t, newRollbackFiler)
+}
+
+func testFilerReadAtWriteAt(t *testing.T, nf newFunc) {
+	f := nf()
+	t.Log(f.Name())
+	defer func() {
+		if err := f.Close(); err != nil {
+			t.Error(err)
+		}
+	}()
+
+	if _, ok := f.(*RollbackFiler); ok {
+		if err := f.BeginUpdate(); err != nil {
+			t.Fatal(err)
+		}
+
+		defer func() {
+			if err := f.EndUpdate(); err != nil {
+				t.Error(err)
+			}
+		}()
+	}
+
+	const (
+		N = 1 << 16
+		M = 2e2
+	)
+
+	s := make([]byte, N)
+	e := make([]byte, N)
+	rnd := rand.New(rand.NewSource(42))
+	for i := range e {
+		s[i] = byte(rnd.Intn(256))
+	}
+	n2 := 0
+	for i := 0; i < M; i++ {
+		var from, to int
+		for {
+			from = rnd.Intn(N)
+			to = rnd.Intn(N)
+			if from != to {
+				break
+			}
+		}
+		if from > to {
+			from, to = to, from
+		}
+		for i := range s[from:to] {
+			s[from+i] = byte(rnd.Intn(256))
+		}
+		copy(e[from:to], s[from:to])
+		if to > n2 {
+			n2 = to
+		}
+		n, err := f.WriteAt(s[from:to], int64(from))
+		if err != nil {
+			t.Error(err)
+			return
+		}
+
+		if g, e := n, to-from; g != e {
+			t.Error(g, e)
+			return
+		}
+	}
+
+	fsz, err := f.Size()
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	if g, e := fsz, int64(n2); g != e {
+		t.Error(g, e)
+		return
+	}
+
+	b := make([]byte, n2)
+	for i := 0; i <= M; i++ {
+		from := rnd.Intn(n2)
+		to := rnd.Intn(n2)
+		if from > to {
+			from, to = to, from
+		}
+		if i == M {
+			from, to = 0, n2
+		}
+		n, err := f.ReadAt(b[from:to], int64(from))
+		if err != nil && (!fileutil.IsEOF(err) && n != 0) {
+			fsz, err = f.Size()
+			if err != nil {
+				t.Error(err)
+				return
+			}
+
+			t.Error(fsz, from, to, err)
+			return
+		}
+
+		if g, e := n, to-from; g != e {
+			t.Error(g, e)
+			return
+		}
+
+		if g, e := b[from:to], e[from:to]; !bytes.Equal(g, e) {
+			if x, ok := f.(*MemFiler); ok {
+				for i := int64(0); i <= 3; i++ {
+					t.Logf("pg %d\n----\n%s", i, hex.Dump(x.m[i][:]))
+				}
+			}
+			t.Errorf(
+				"i %d from %d to %d len(g) %d len(e) %d\n---- got ----\n%s\n---- exp ----\n%s",
+				i, from, to, len(g), len(e), hex.Dump(g), hex.Dump(e),
+			)
+			return
+		}
+	}
+
+	mf, ok := f.(*MemFiler)
+	if !ok {
+		return
+	}
+
+	buf := &bytes.Buffer{}
+	if _, err := mf.WriteTo(buf); err != nil {
+		t.Error(err)
+		return
+	}
+
+	if g, e := buf.Bytes(), e[:n2]; !bytes.Equal(g, e) {
+		t.Errorf("\nlen %d\n%s\nlen %d\n%s", len(g), hex.Dump(g), len(e), hex.Dump(e))
+		return
+	}
+
+	if err := mf.Truncate(0); err != nil {
+		t.Error(err)
+		return
+	}
+
+	if _, err := mf.ReadFrom(buf); err != nil {
+		t.Error(err)
+		return
+	}
+
+	roundTrip := make([]byte, n2)
+	if n, err := mf.ReadAt(roundTrip, 0); err != nil && n == 0 {
+		t.Error(err)
+		return
+	}
+
+	if g, e := roundTrip, e[:n2]; !bytes.Equal(g, e) {
+		t.Errorf("\nlen %d\n%s\nlen %d\n%s", len(g), hex.Dump(g), len(e), hex.Dump(e))
+		return
+	}
+}
+
+func TestInnerFiler(t *testing.T) {
+	testInnerFiler(t, newFileFiler)
+	testInnerFiler(t, newOSFileFiler)
+	testInnerFiler(t, newMemFiler)
+	testInnerFiler(t, nwBitFiler)
+	testInnerFiler(t, newRollbackFiler)
+}
+
+func testInnerFiler(t *testing.T, nf newFunc) {
+	const (
+		HDR_SIZE = 42
+		LONG_OFF = 3330
+	)
+	outer := nf()
+	t.Log(outer.Name())
+	inner := NewInnerFiler(outer, HDR_SIZE)
+	defer func() {
+		if err := outer.Close(); err != nil {
+			t.Error(err)
+		}
+	}()
+
+	if _, ok := outer.(*RollbackFiler); ok {
+		if err := outer.BeginUpdate(); err != nil {
+			t.Fatal(err)
+		}
+
+		defer func() {
+			if err := outer.EndUpdate(); err != nil {
+				t.Error(err)
+			}
+		}()
+	}
+
+	b := []byte{2, 5, 11}
+	n, err := inner.WriteAt(b, -1)
+	if err == nil {
+		t.Error("unexpected success")
+		return
+	}
+
+	n, err = inner.ReadAt(make([]byte, 10), -1)
+	if err == nil {
+		t.Error("unexpected success")
+		return
+	}
+
+	n, err = inner.WriteAt(b, 0)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	if g, e := n, len(b); g != e {
+		t.Error(g, e)
+		return
+	}
+
+	osz, err := outer.Size()
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	if g, e := osz, int64(HDR_SIZE+3); g != e {
+		t.Error(g, e)
+		return
+	}
+
+	isz, err := inner.Size()
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	if g, e := isz, int64(3); g != e {
+		t.Error(g, e)
+		return
+	}
+
+	rbuf := make([]byte, 3)
+	if n, err = outer.ReadAt(rbuf, 0); err != nil && n == 0 {
+		t.Error(err)
+		return
+	}
+
+	if g, e := n, len(rbuf); g != e {
+		t.Error(g, e)
+		return
+	}
+
+	if g, e := rbuf, make([]byte, 3); !bytes.Equal(g, e) {
+		t.Error(g, e)
+	}
+
+	rbuf = make([]byte, 3)
+	if n, err = outer.ReadAt(rbuf, HDR_SIZE); err != nil && n == 0 {
+		t.Error(err)
+		return
+	}
+
+	if g, e := n, len(rbuf); g != e {
+		t.Error(g, e)
+		return
+	}
+
+	if g, e := rbuf, []byte{2, 5, 11}; !bytes.Equal(g, e) {
+		t.Error(g, e)
+	}
+
+	rbuf = make([]byte, 3)
+	if n, err = inner.ReadAt(rbuf, 0); err != nil && n == 0 {
+		t.Error(err)
+		return
+	}
+
+	if g, e := n, len(rbuf); g != e {
+		t.Error(g, e)
+		return
+	}
+
+	if g, e := rbuf, []byte{2, 5, 11}; !bytes.Equal(g, e) {
+		t.Error(g, e)
+	}
+
+	b = []byte{22, 55, 111}
+	if n, err = inner.WriteAt(b, LONG_OFF); err != nil {
+		t.Error(err)
+		return
+	}
+
+	if g, e := n, len(b); g != e {
+		t.Error(g, e)
+		return
+	}
+
+	osz, err = outer.Size()
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	if g, e := osz, int64(HDR_SIZE+LONG_OFF+3); g != e {
+		t.Error(g, e)
+		return
+	}
+
+	isz, err = inner.Size()
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	if g, e := isz, int64(LONG_OFF+3); g != e {
+		t.Error(g, e)
+		return
+	}
+
+	rbuf = make([]byte, 3)
+	if n, err = outer.ReadAt(rbuf, HDR_SIZE+LONG_OFF); err != nil && n == 0 {
+		t.Error(err)
+		return
+	}
+
+	if g, e := n, len(rbuf); g != e {
+		t.Error(g, e)
+		return
+	}
+
+	if g, e := rbuf, []byte{22, 55, 111}; !bytes.Equal(g, e) {
+		t.Error(g, e)
+	}
+
+	rbuf = make([]byte, 3)
+	if n, err = inner.ReadAt(rbuf, LONG_OFF); err != nil && n == 0 {
+		t.Error(err)
+		return
+	}
+
+	if g, e := n, len(rbuf); g != e {
+		t.Error(g, e)
+		return
+	}
+
+	if g, e := rbuf, []byte{22, 55, 111}; !bytes.Equal(g, e) {
+		t.Error(g, e)
+		return
+	}
+
+	if err = inner.Truncate(1); err != nil {
+		t.Error(err)
+		return
+	}
+
+	isz, err = inner.Size()
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	if g, e := isz, int64(1); g != e {
+		t.Error(g, e)
+		return
+	}
+
+	osz, err = outer.Size()
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	if g, e := osz, int64(HDR_SIZE+1); g != e {
+		t.Error(g, e)
+		return
+	}
+}
+
+func TestFileReadAtHole(t *testing.T) {
+	testFileReadAtHole(t, newFileFiler)
+	testFileReadAtHole(t, newOSFileFiler)
+	testFileReadAtHole(t, newMemFiler)
+	testFileReadAtHole(t, nwBitFiler)
+	testFileReadAtHole(t, newRollbackFiler)
+}
+
+func testFileReadAtHole(t *testing.T, nf newFunc) {
+	f := nf()
+	t.Log(f.Name())
+	defer func() {
+		if err := f.Close(); err != nil {
+			t.Error(err)
+		}
+	}()
+
+	if _, ok := f.(*RollbackFiler); ok {
+		if err := f.BeginUpdate(); err != nil {
+			t.Fatal(err)
+		}
+
+		defer func() {
+			if err := f.EndUpdate(); err != nil {
+				t.Error(err)
+			}
+		}()
+	}
+
+	n, err := f.WriteAt([]byte{1}, 40000)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	if n != 1 {
+		t.Error(n)
+		return
+	}
+
+	n, err = f.ReadAt(make([]byte, 1000), 20000)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	if n != 1000 {
+		t.Error(n)
+		return
+	}
+}
+
+func BenchmarkMemFilerWrSeq(b *testing.B) {
+	b.StopTimer()
+	buf := make([]byte, filerTestChunkSize)
+	for i := range buf {
+		buf[i] = byte(rand.Int())
+	}
+	f := newMemFiler()
+	runtime.GC()
+	b.StartTimer()
+	var ofs int64
+	for i := 0; i < b.N; i++ {
+		_, err := f.WriteAt(buf, ofs)
+		if err != nil {
+			b.Fatal(err)
+		}
+
+		ofs = (ofs + filerTestChunkSize) % filerTotalSize
+	}
+}
+
+func BenchmarkMemFilerRdSeq(b *testing.B) {
+	b.StopTimer()
+	buf := make([]byte, filerTestChunkSize)
+	for i := range buf {
+		buf[i] = byte(rand.Int())
+	}
+	f := newMemFiler()
+	var ofs int64
+	for i := 0; i < b.N; i++ {
+		_, err := f.WriteAt(buf, ofs)
+		if err != nil {
+			b.Fatal(err)
+		}
+
+		ofs = (ofs + filerTestChunkSize) % filerTotalSize
+	}
+	runtime.GC()
+	b.StartTimer()
+	ofs = 0
+	for i := 0; i < b.N; i++ {
+		n, err := f.ReadAt(buf, ofs)
+		if err != nil && n == 0 {
+			b.Fatal(err)
+		}
+
+		ofs = (ofs + filerTestChunkSize) % filerTotalSize
+	}
+}
+
+func BenchmarkMemFilerWrRand(b *testing.B) {
+	b.StopTimer()
+	rng := rand.New(rand.NewSource(42))
+	f := newMemFiler()
+	var bytes int64
+
+	var ofs, runs []int
+	for i := 0; i < b.N; i++ {
+		ofs = append(ofs, rng.Intn(1<<31-1))
+		runs = append(runs, rng.Intn(1<<31-1)%(2*pgSize))
+	}
+	data := make([]byte, 2*pgSize)
+	for i := range data {
+		data[i] = byte(rng.Int())
+	}
+
+	runtime.GC()
+	b.StartTimer()
+	for i, v := range ofs {
+		n := runs[i]
+		bytes += int64(n)
+		f.WriteAt(data[:n], int64(v))
+	}
+	b.StopTimer()
+}
+
+func BenchmarkMemFilerRdRand(b *testing.B) {
+	b.StopTimer()
+	rng := rand.New(rand.NewSource(42))
+	f := newMemFiler()
+	var bytes int64
+
+	var ofs, runs []int
+	for i := 0; i < b.N; i++ {
+		ofs = append(ofs, rng.Intn(1<<31-1))
+		runs = append(runs, rng.Intn(1<<31-1)%(2*pgSize))
+	}
+	data := make([]byte, 2*pgSize)
+	for i := range data {
+		data[i] = byte(rng.Int())
+	}
+
+	for i, v := range ofs {
+		n := runs[i]
+		bytes += int64(n)
+		f.WriteAt(data[:n], int64(v))
+	}
+
+	runtime.GC()
+	b.StartTimer()
+	for _, v := range ofs {
+		f.ReadAt(data, int64(v))
+	}
+	b.StopTimer()
+}

Godeps/_workspace/src/github.com/cznic/exp/lldb/gb.go

@@ -0,0 +1,812 @@
+// Copyright 2014 The lldb Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Utilities to encode/decode and collate Go predeclared scalar types (and the
+// typeless nil and []byte).  The encoding format is a variation of the one
+// used by the "encoding/gob" package.
+
+package lldb
+
+import (
+	"bytes"
+	"fmt"
+	"math"
+
+	"github.com/coreos/rocket/Godeps/_workspace/src/github.com/cznic/mathutil"
+)
+
+const (
+	gbNull     = iota // 0x00
+	gbFalse           // 0x01
+	gbTrue            // 0x02
+	gbFloat0          // 0x03
+	gbFloat1          // 0x04
+	gbFloat2          // 0x05
+	gbFloat3          // 0x06
+	gbFloat4          // 0x07
+	gbFloat5          // 0x08
+	gbFloat6          // 0x09
+	gbFloat7          // 0x0a
+	gbFloat8          // 0x0b
+	gbComplex0        // 0x0c
+	gbComplex1        // 0x0d
+	gbComplex2        // 0x0e
+	gbComplex3        // 0x0f
+	gbComplex4        // 0x10
+	gbComplex5        // 0x11
+	gbComplex6        // 0x12
+	gbComplex7        // 0x13
+	gbComplex8        // 0x14
+	gbBytes00         // 0x15
+	gbBytes01         // 0x16
+	gbBytes02         // 0x17
+	gbBytes03         // 0x18
+	gbBytes04         // 0x19
+	gbBytes05         // 0x1a
+	gbBytes06         // 0x1b
+	gbBytes07         // 0x1c
+	gbBytes08         // 0x1d
+	gbBytes09         // 0x1e
+	gbBytes10         // 0x1f
+	gbBytes11         // 0x20
+	gbBytes12         // 0x21
+	gbBytes13         // 0x22
+	gbBytes14         // 0x23
+	gbBytes15         // 0x24
+	gbBytes16         // 0x25
+	gbBytes17         // Ox26
+	gbBytes1          // 0x27
+	gbBytes2          // 0x28: Offset by one to allow 64kB sized []byte.
+	gbString00        // 0x29
+	gbString01        // 0x2a
+	gbString02        // 0x2b
+	gbString03        // 0x2c
+	gbString04        // 0x2d
+	gbString05        // 0x2e
+	gbString06        // 0x2f
+	gbString07        // 0x30
+	gbString08        // 0x31
+	gbString09        // 0x32
+	gbString10        // 0x33
+	gbString11        // 0x34
+	gbString12        // 0x35
+	gbString13        // 0x36
+	gbString14        // 0x37
+	gbString15        // 0x38
+	gbString16        // 0x39
+	gbString17        // 0x3a
+	gbString1         // 0x3b
+	gbString2         // 0x3c
+	gbUintP1          // 0x3d
+	gbUintP2          // 0x3e
+	gbUintP3          // 0x3f
+	gbUintP4          // 0x40
+	gbUintP5          // 0x41
+	gbUintP6          // 0x42
+	gbUintP7          // 0x43
+	gbUintP8          // 0x44
+	gbIntM8           // 0x45
+	gbIntM7           // 0x46
+	gbIntM6           // 0x47
+	gbIntM5           // 0x48
+	gbIntM4           // 0x49
+	gbIntM3           // 0x4a
+	gbIntM2           // 0x4b
+	gbIntM1           // 0x4c
+	gbIntP1           // 0x4d
+	gbIntP2           // 0x4e
+	gbIntP3           // 0x4f
+	gbIntP4           // 0x50
+	gbIntP5           // 0x51
+	gbIntP6           // 0x52
+	gbIntP7           // 0x53
+	gbIntP8           // 0x54
+	gbInt0            // 0x55
+
+	gbIntMax = 255 - gbInt0 // 0xff == 170
+)
+
+// EncodeScalars encodes a vector of predeclared scalar type values to a
+// []byte, making it suitable to store it as a "record" in a DB or to use it as
+// a key of a BTree.
+func EncodeScalars(scalars ...interface{}) (b []byte, err error) {
+	for _, scalar := range scalars {
+		switch x := scalar.(type) {
+		default:
+			return nil, &ErrINVAL{"EncodeScalars: unsupported type", fmt.Sprintf("%T in `%#v`", x, scalars)}
+
+		case nil:
+			b = append(b, gbNull)
+
+		case bool:
+			switch x {
+			case false:
+				b = append(b, gbFalse)
+			case true:
+				b = append(b, gbTrue)
+			}
+
+		case float32:
+			encFloat(float64(x), &b)
+		case float64:
+			encFloat(x, &b)
+
+		case complex64:
+			encComplex(complex128(x), &b)
+		case complex128:
+			encComplex(x, &b)
+
+		case string:
+			n := len(x)
+			if n <= 17 {
+				b = append(b, byte(gbString00+n))
+				b = append(b, []byte(x)...)
+				break
+			}
+
+			if n > 65535 {
+				return nil, fmt.Errorf("EncodeScalars: cannot encode string of length %d (limit 65536)", n)
+			}
+
+			pref := byte(gbString1)
+			if n > 255 {
+				pref++
+			}
+			b = append(b, pref)
+			encUint0(uint64(n), &b)
+			b = append(b, []byte(x)...)
+
+		case int8:
+			encInt(int64(x), &b)
+		case int16:
+			encInt(int64(x), &b)
+		case int32:
+			encInt(int64(x), &b)
+		case int64:
+			encInt(x, &b)
+		case int:
+			encInt(int64(x), &b)
+
+		case uint8:
+			encUint(uint64(x), &b)
+		case uint16:
+			encUint(uint64(x), &b)
+		case uint32:
+			encUint(uint64(x), &b)
+		case uint64:
+			encUint(x, &b)
+		case uint:
+			encUint(uint64(x), &b)
+		case []byte:
+			n := len(x)
+			if n <= 17 {
+				b = append(b, byte(gbBytes00+n))
+				b = append(b, []byte(x)...)
+				break
+			}
+
+			if n > 655356 {
+				return nil, fmt.Errorf("EncodeScalars: cannot encode []byte of length %d (limit 65536)", n)
+			}
+
+			pref := byte(gbBytes1)
+			if n > 255 {
+				pref++
+			}
+			b = append(b, pref)
+			if n <= 255 {
+				b = append(b, byte(n))
+			} else {
+				n--
+				b = append(b, byte(n>>8), byte(n))
+			}
+			b = append(b, x...)
+		}
+	}
+	return
+}
+
+func encComplex(f complex128, b *[]byte) {
+	encFloatPrefix(gbComplex0, real(f), b)
+	encFloatPrefix(gbComplex0, imag(f), b)
+}
+
+func encFloatPrefix(prefix byte, f float64, b *[]byte) {
+	u := math.Float64bits(f)
+	var n uint64
+	for i := 0; i < 8; i++ {
+		n <<= 8
+		n |= u & 0xFF
+		u >>= 8
+	}
+	bits := mathutil.BitLenUint64(n)
+	if bits == 0 {
+		*b = append(*b, prefix)
+		return
+	}
+
+	// 0 1 2 3 4 5 6 7 8 9
+	// . 1 1 1 1 1 1 1 1 2
+	encUintPrefix(prefix+1+byte((bits-1)>>3), n, b)
+}
+
+func encFloat(f float64, b *[]byte) {
+	encFloatPrefix(gbFloat0, f, b)
+}
+
+func encUint0(n uint64, b *[]byte) {
+	switch {
+	case n <= 0xff:
+		*b = append(*b, byte(n))
+	case n <= 0xffff:
+		*b = append(*b, byte(n>>8), byte(n))
+	case n <= 0xffffff:
+		*b = append(*b, byte(n>>16), byte(n>>8), byte(n))
+	case n <= 0xffffffff:
+		*b = append(*b, byte(n>>24), byte(n>>16), byte(n>>8), byte(n))
+	case n <= 0xffffffffff:
+		*b = append(*b, byte(n>>32), byte(n>>24), byte(n>>16), byte(n>>8), byte(n))
+	case n <= 0xffffffffffff:
+		*b = append(*b, byte(n>>40), byte(n>>32), byte(n>>24), byte(n>>16), byte(n>>8), byte(n))
+	case n <= 0xffffffffffffff:
+		*b = append(*b, byte(n>>48), byte(n>>40), byte(n>>32), byte(n>>24), byte(n>>16), byte(n>>8), byte(n))
+	case n <= math.MaxUint64:
+		*b = append(*b, byte(n>>56), byte(n>>48), byte(n>>40), byte(n>>32), byte(n>>24), byte(n>>16), byte(n>>8), byte(n))
+	}
+}
+
+func encUintPrefix(prefix byte, n uint64, b *[]byte) {
+	*b = append(*b, prefix)
+	encUint0(n, b)
+}
+
+func encUint(n uint64, b *[]byte) {
+	bits := mathutil.Max(1, mathutil.BitLenUint64(n))
+	encUintPrefix(gbUintP1+byte((bits-1)>>3), n, b)
+}
+
+func encInt(n int64, b *[]byte) {
+	switch {
+	case n < -0x100000000000000:
+		*b = append(*b, byte(gbIntM8), byte(n>>56), byte(n>>48), byte(n>>40), byte(n>>32), byte(n>>24), byte(n>>16), byte(n>>8), byte(n))
+	case n < -0x1000000000000:
+		*b = append(*b, byte(gbIntM7), byte(n>>48), byte(n>>40), byte(n>>32), byte(n>>24), byte(n>>16), byte(n>>8), byte(n))
+	case n < -0x10000000000:
+		*b = append(*b, byte(gbIntM6), byte(n>>40), byte(n>>32), byte(n>>24), byte(n>>16), byte(n>>8), byte(n))
+	case n < -0x100000000:
+		*b = append(*b, byte(gbIntM5), byte(n>>32), byte(n>>24), byte(n>>16), byte(n>>8), byte(n))
+	case n < -0x1000000:
+		*b = append(*b, byte(gbIntM4), byte(n>>24), byte(n>>16), byte(n>>8), byte(n))
+	case n < -0x10000:
+		*b = append(*b, byte(gbIntM3), byte(n>>16), byte(n>>8), byte(n))
+	case n < -0x100:
+		*b = append(*b, byte(gbIntM2), byte(n>>8), byte(n))
+	case n < 0:
+		*b = append(*b, byte(gbIntM1), byte(n))
+	case n <= gbIntMax:
+		*b = append(*b, byte(gbInt0+n))
+	case n <= 0xff:
+		*b = append(*b, gbIntP1, byte(n))
+	case n <= 0xffff:
+		*b = append(*b, gbIntP2, byte(n>>8), byte(n))
+	case n <= 0xffffff:
+		*b = append(*b, gbIntP3, byte(n>>16), byte(n>>8), byte(n))
+	case n <= 0xffffffff:
+		*b = append(*b, gbIntP4, byte(n>>24), byte(n>>16), byte(n>>8), byte(n))
+	case n <= 0xffffffffff:
+		*b = append(*b, gbIntP5, byte(n>>32), byte(n>>24), byte(n>>16), byte(n>>8), byte(n))
+	case n <= 0xffffffffffff:
+		*b = append(*b, gbIntP6, byte(n>>40), byte(n>>32), byte(n>>24), byte(n>>16), byte(n>>8), byte(n))
+	case n <= 0xffffffffffffff:
+		*b = append(*b, gbIntP7, byte(n>>48), byte(n>>40), byte(n>>32), byte(n>>24), byte(n>>16), byte(n>>8), byte(n))
+	case n <= 0x7fffffffffffffff:
+		*b = append(*b, gbIntP8, byte(n>>56), byte(n>>48), byte(n>>40), byte(n>>32), byte(n>>24), byte(n>>16), byte(n>>8), byte(n))
+	}
+}
+
+func decodeFloat(b []byte) float64 {
+	var u uint64
+	for i, v := range b {
+		u |= uint64(v) << uint((i+8-len(b))*8)
+	}
+	return math.Float64frombits(u)
+}
+
+// DecodeScalars decodes a []byte produced by EncodeScalars.
+func DecodeScalars(b []byte) (scalars []interface{}, err error) {
+	b0 := b
+	for len(b) != 0 {
+		switch tag := b[0]; tag {
+		//default:
+		//return nil, fmt.Errorf("tag %d(%#x) not supported", b[0], b[0])
+		case gbNull:
+			scalars = append(scalars, nil)
+			b = b[1:]
+		case gbFalse:
+			scalars = append(scalars, false)
+			b = b[1:]
+		case gbTrue:
+			scalars = append(scalars, true)
+			b = b[1:]
+		case gbFloat0:
+			scalars = append(scalars, 0.0)
+			b = b[1:]
+		case gbFloat1, gbFloat2, gbFloat3, gbFloat4, gbFloat5, gbFloat6, gbFloat7, gbFloat8:
+			n := 1 + int(tag) - gbFloat0
+			if len(b) < n-1 {
+				goto corrupted
+			}
+
+			scalars = append(scalars, decodeFloat(b[1:n]))
+			b = b[n:]
+		case gbComplex0, gbComplex1, gbComplex2, gbComplex3, gbComplex4, gbComplex5, gbComplex6, gbComplex7, gbComplex8:
+			n := 1 + int(tag) - gbComplex0
+			if len(b) < n-1 {
+				goto corrupted
+			}
+
+			re := decodeFloat(b[1:n])
+			b = b[n:]
+
+			if len(b) == 0 {
+				goto corrupted
+			}
+
+			tag = b[0]
+			if tag < gbComplex0 || tag > gbComplex8 {
+				goto corrupted
+			}
+
+			n = 1 + int(tag) - gbComplex0
+			if len(b) < n-1 {
+				goto corrupted
+			}
+
+			scalars = append(scalars, complex(re, decodeFloat(b[1:n])))
+			b = b[n:]
+		case gbBytes00, gbBytes01, gbBytes02, gbBytes03, gbBytes04,
+			gbBytes05, gbBytes06, gbBytes07, gbBytes08, gbBytes09,
+			gbBytes10, gbBytes11, gbBytes12, gbBytes13, gbBytes14,
+			gbBytes15, gbBytes16, gbBytes17:
+			n := int(tag - gbBytes00)
+			if len(b) < n+1 {
+				goto corrupted
+			}
+
+			scalars = append(scalars, append([]byte(nil), b[1:n+1]...))
+			b = b[n+1:]
+		case gbBytes1:
+			if len(b) < 2 {
+				goto corrupted
+			}
+
+			n := int(b[1])
+			b = b[2:]
+			if len(b) < n {
+				goto corrupted
+			}
+
+			scalars = append(scalars, append([]byte(nil), b[:n]...))
+			b = b[n:]
+		case gbBytes2:
+			if len(b) < 3 {
+				goto corrupted
+			}
+
+			n := int(b[1])<<8 | int(b[2]) + 1
+			b = b[3:]
+			if len(b) < n {
+				goto corrupted
+			}
+
+			scalars = append(scalars, append([]byte(nil), b[:n]...))
+			b = b[n:]
+		case gbString00, gbString01, gbString02, gbString03, gbString04,
+			gbString05, gbString06, gbString07, gbString08, gbString09,
+			gbString10, gbString11, gbString12, gbString13, gbString14,
+			gbString15, gbString16, gbString17:
+			n := int(tag - gbString00)
+			if len(b) < n+1 {
+				goto corrupted
+			}
+
+			scalars = append(scalars, string(b[1:n+1]))
+			b = b[n+1:]
+		case gbString1:
+			if len(b) < 2 {
+				goto corrupted
+			}
+
+			n := int(b[1])
+			b = b[2:]
+			if len(b) < n {
+				goto corrupted
+			}
+
+			scalars = append(scalars, string(b[:n]))
+			b = b[n:]
+		case gbString2:
+			if len(b) < 3 {
+				goto corrupted
+			}
+
+			n := int(b[1])<<8 | int(b[2])
+			b = b[3:]
+			if len(b) < n {
+				goto corrupted
+			}
+
+			scalars = append(scalars, string(b[:n]))
+			b = b[n:]
+		case gbUintP1, gbUintP2, gbUintP3, gbUintP4, gbUintP5, gbUintP6, gbUintP7, gbUintP8:
+			b = b[1:]
+			n := 1 + int(tag) - gbUintP1
+			if len(b) < n {
+				goto corrupted
+			}
+
+			var u uint64
+			for _, v := range b[:n] {
+				u = u<<8 | uint64(v)
+			}
+			scalars = append(scalars, u)
+			b = b[n:]
+		case gbIntM8, gbIntM7, gbIntM6, gbIntM5, gbIntM4, gbIntM3, gbIntM2, gbIntM1:
+			b = b[1:]
+			n := 8 - (int(tag) - gbIntM8)
+			if len(b) < n {
+				goto corrupted
+			}
+			u := uint64(math.MaxUint64)
+			for _, v := range b[:n] {
+				u = u<<8 | uint64(v)
+			}
+			scalars = append(scalars, int64(u))
+			b = b[n:]
+		case gbIntP1, gbIntP2, gbIntP3, gbIntP4, gbIntP5, gbIntP6, gbIntP7, gbIntP8:
+			b = b[1:]
+			n := 1 + int(tag) - gbIntP1
+			if len(b) < n {
+				goto corrupted
+			}
+
+			i := int64(0)
+			for _, v := range b[:n] {
+				i = i<<8 | int64(v)
+			}
+			scalars = append(scalars, i)
+			b = b[n:]
+		default:
+			scalars = append(scalars, int64(b[0])-gbInt0)
+			b = b[1:]
+		}
+	}
+	return append([]interface{}(nil), scalars...), nil
+
+corrupted:
+	return nil, &ErrDecodeScalars{append([]byte(nil), b0...), len(b0) - len(b)}
+}
+
+func collateComplex(x, y complex128) int {
+	switch rx, ry := real(x), real(y); {
+	case rx < ry:
+		return -1
+	case rx == ry:
+		switch ix, iy := imag(x), imag(y); {
+		case ix < iy:
+			return -1
+		case ix == iy:
+			return 0
+		case ix > iy:
+			return 1
+		}
+	}
+	//case rx > ry:
+	return 1
+}
+
+func collateFloat(x, y float64) int {
+	switch {
+	case x < y:
+		return -1
+	case x == y:
+		return 0
+	}
+	//case x > y:
+	return 1
+}
+
+func collateInt(x, y int64) int {
+	switch {
+	case x < y:
+		return -1
+	case x == y:
+		return 0
+	}
+	//case x > y:
+	return 1
+}
+
+func collateUint(x, y uint64) int {
+	switch {
+	case x < y:
+		return -1
+	case x == y:
+		return 0
+	}
+	//case x > y:
+	return 1
+}
+
+func collateIntUint(x int64, y uint64) int {
+	if y > math.MaxInt64 {
+		return -1
+	}
+
+	return collateInt(x, int64(y))
+}
+
+func collateUintInt(x uint64, y int64) int {
+	return -collateIntUint(y, x)
+}
+
+func collateType(i interface{}) (r interface{}, err error) {
+	switch x := i.(type) {
+	default:
+		return nil, fmt.Errorf("invalid collate type %T", x)
+	case nil:
+		return i, nil
+	case bool:
+		return i, nil
+	case int8:
+		return int64(x), nil
+	case int16:
+		return int64(x), nil
+	case int32:
+		return int64(x), nil
+	case int64:
+		return i, nil
+	case int:
+		return int64(x), nil
+	case uint8:
+		return uint64(x), nil
+	case uint16:
+		return uint64(x), nil
+	case uint32:
+		return uint64(x), nil
+	case uint64:
+		return i, nil
+	case uint:
+		return uint64(x), nil
+	case float32:
+		return float64(x), nil
+	case float64:
+		return i, nil
+	case complex64:
+		return complex128(x), nil
+	case complex128:
+		return i, nil
+	case []byte:
+		return i, nil
+	case string:
+		return i, nil
+	}
+}
+
+// Collate collates two arrays of Go predeclared scalar types (and the typeless
+// nil or []byte). If any other type appears in x or y, Collate will return a
+// non nil error.  String items are collated using strCollate or lexically
+// byte-wise (as when using Go comparison operators) when strCollate is nil.
+// []byte items are collated using bytes.Compare.
+//
+// Collate returns:
+//
+// 	-1 if x <  y
+// 	 0 if x == y
+// 	+1 if x >  y
+//
+// The same value as defined above must be returned from strCollate.
+//
+// The "outer" ordering is: nil, bool, number, []byte, string. IOW, nil is
+// "smaller" than anything else except other nil, numbers collate before
+// []byte, []byte collate before strings, etc.
+//
+// Integers and real numbers collate as expected in math. However, complex
+// numbers are not ordered in Go. Here the ordering is defined: Complex numbers
+// are in comparison considered first only by their real part. Iff the result
+// is equality then the imaginary part is used to determine the ordering. In
+// this "second order" comparing, integers and real numbers are considered as
+// complex numbers with a zero imaginary part.
+func Collate(x, y []interface{}, strCollate func(string, string) int) (r int, err error) {
+	nx, ny := len(x), len(y)
+
+	switch {
+	case nx == 0 && ny != 0:
+		return -1, nil
+	case nx == 0 && ny == 0:
+		return 0, nil
+	case nx != 0 && ny == 0:
+		return 1, nil
+	}
+
+	r = 1
+	if nx > ny {
+		x, y, r = y, x, -r
+	}
+
+	var c int
+	for i, xi0 := range x {
+		yi0 := y[i]
+		xi, err := collateType(xi0)
+		if err != nil {
+			return 0, err
+		}
+
+		yi, err := collateType(yi0)
+		if err != nil {
+			return 0, err
+		}
+
+		switch x := xi.(type) {
+		default:
+			panic(fmt.Errorf("internal error: %T", x))
+
+		case nil:
+			switch yi.(type) {
+			case nil:
+				// nop
+			default:
+				return -r, nil
+			}
+
+		case bool:
+			switch y := yi.(type) {
+			case nil:
+				return r, nil
+			case bool:
+				switch {
+				case !x && y:
+					return -r, nil
+				case x == y:
+					// nop
+				case x && !y:
+					return r, nil
+				}
+			default:
+				return -r, nil
+			}
+
+		case int64:
+			switch y := yi.(type) {
+			case nil, bool:
+				return r, nil
+			case int64:
+				c = collateInt(x, y)
+			case uint64:
+				c = collateIntUint(x, y)
+			case float64:
+				c = collateFloat(float64(x), y)
+			case complex128:
+				c = collateComplex(complex(float64(x), 0), y)
+			case []byte:
+				return -r, nil
+			case string:
+				return -r, nil
+			}
+
+			if c != 0 {
+				return c * r, nil
+			}
+
+		case uint64:
+			switch y := yi.(type) {
+			case nil, bool:
+				return r, nil
+			case int64:
+				c = collateUintInt(x, y)
+			case uint64:
+				c = collateUint(x, y)
+			case float64:
+				c = collateFloat(float64(x), y)
+			case complex128:
+				c = collateComplex(complex(float64(x), 0), y)
+			case []byte:
+				return -r, nil
+			case string:
+				return -r, nil
+			}
+
+			if c != 0 {
+				return c * r, nil
+			}
+
+		case float64:
+			switch y := yi.(type) {
+			case nil, bool:
+				return r, nil
+			case int64:
+				c = collateFloat(x, float64(y))
+			case uint64:
+				c = collateFloat(x, float64(y))
+			case float64:
+				c = collateFloat(x, y)
+			case complex128:
+				c = collateComplex(complex(x, 0), y)
+			case []byte:
+				return -r, nil
+			case string:
+				return -r, nil
+			}
+
+			if c != 0 {
+				return c * r, nil
+			}
+
+		case complex128:
+			switch y := yi.(type) {
+			case nil, bool:
+				return r, nil
+			case int64:
+				c = collateComplex(x, complex(float64(y), 0))
+			case uint64:
+				c = collateComplex(x, complex(float64(y), 0))
+			case float64:
+				c = collateComplex(x, complex(y, 0))
+			case complex128:
+				c = collateComplex(x, y)
+			case []byte:
+				return -r, nil
+			case string:
+				return -r, nil
+			}
+
+			if c != 0 {
+				return c * r, nil
+			}
+
+		case []byte:
+			switch y := yi.(type) {
+			case nil, bool, int64, uint64, float64, complex128:
+				return r, nil
+			case []byte:
+				c = bytes.Compare(x, y)
+			case string:
+				return -r, nil
+			}
+
+			if c != 0 {
+				return c * r, nil
+			}
+
+		case string:
+			switch y := yi.(type) {
+			case nil, bool, int64, uint64, float64, complex128:
+				return r, nil
+			case []byte:
+				return r, nil
+			case string:
+				switch {
+				case strCollate != nil:
+					c = strCollate(x, y)
+				case x < y:
+					return -r, nil
+				case x == y:
+					c = 0
+				case x > y:
+					return r, nil
+				}
+			}
+
+			if c != 0 {
+				return c * r, nil
+			}
+		}
+	}
+
+	if nx == ny {
+		return 0, nil
+	}
+
+	return -r, nil
+}

Godeps/_workspace/src/github.com/cznic/exp/lldb/gb_test.go

@@ -0,0 +1,364 @@
+// Copyright 2014 The lldb Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Utilities to encode/decode and collate Go predeclared scalar types.  The
+// encoding format reused the one used by the "encoding/gob" package.
+
+package lldb
+
+import (
+	"bytes"
+	"math"
+	"testing"
+)
+
+const s256 = "" +
+	"0123456789abcdef" +
+	"0123456789abcdef" +
+	"0123456789abcdef" +
+	"0123456789abcdef" +
+	"0123456789abcdef" +
+	"0123456789abcdef" +
+	"0123456789abcdef" +
+	"0123456789abcdef" +
+	"0123456789abcdef" +
+	"0123456789abcdef" +
+	"0123456789abcdef" +
+	"0123456789abcdef" +
+	"0123456789abcdef" +
+	"0123456789abcdef" +
+	"0123456789abcdef" +
+	"0123456789abcdef"
+
+func TestEncodeDecodeScalars(t *testing.T) {
+	table := []struct{ v, exp interface{} }{
+		{nil, "00"},
+		{false, "01"},
+		{true, "02"},
+		{math.Float64frombits(0), []byte{gbFloat0}},
+		{17., []byte{gbFloat2, 0x31, 0x40}},
+		{math.Float64frombits(0x4031320000000000), []byte{gbFloat3, 0x32, 0x31, 0x40}},
+		{math.Float64frombits(0x4031323300000000), []byte{gbFloat4, 0x33, 0x32, 0x31, 0x40}},
+		{math.Float64frombits(0x4031323334000000), []byte{gbFloat5, 0x34, 0x33, 0x32, 0x31, 0x40}},
+		{math.Float64frombits(0x4031323334350000), []byte{gbFloat6, 0x35, 0x34, 0x33, 0x32, 0x31, 0x40}},
+		{math.Float64frombits(0x4031323334353600), []byte{gbFloat7, 0x36, 0x35, 0x34, 0x33, 0x32, 0x31, 0x40}},
+		{math.Float64frombits(0x4031323334353637), []byte{gbFloat8, 0x37, 0x36, 0x35, 0x34, 0x33, 0x32, 0x31, 0x40}},
+		{0 + 0i, []byte{gbComplex0, gbComplex0}},
+		{17 + 17i, []byte{gbComplex2, 0x31, 0x40, gbComplex2, 0x31, 0x40}},
+		{complex(math.Float64frombits(0x4041420000000000), math.Float64frombits(0x4031320000000000)), []byte{gbComplex3, 0x42, 0x41, 0x40, gbComplex3, 0x32, 0x31, 0x40}},
+		{complex(math.Float64frombits(0x4041424300000000), math.Float64frombits(0x4031323300000000)), []byte{gbComplex4, 0x43, 0x42, 0x41, 0x40, gbComplex4, 0x33, 0x32, 0x31, 0x40}},
+		{complex(math.Float64frombits(0x4041424344000000), math.Float64frombits(0x4031323334000000)), []byte{gbComplex5, 0x44, 0x43, 0x42, 0x41, 0x40, gbComplex5, 0x34, 0x33, 0x32, 0x31, 0x40}},
+		{complex(math.Float64frombits(0x4041424344450000), math.Float64frombits(0x4031323334350000)), []byte{gbComplex6, 0x45, 0x44, 0x43, 0x42, 0x41, 0x40, gbComplex6, 0x35, 0x34, 0x33, 0x32, 0x31, 0x40}},
+		{complex(math.Float64frombits(0x4041424344454600), math.Float64frombits(0x4031323334353600)), []byte{gbComplex7, 0x46, 0x45, 0x44, 0x43, 0x42, 0x41, 0x40, gbComplex7, 0x36, 0x35, 0x34, 0x33, 0x32, 0x31, 0x40}},
+		{complex(math.Float64frombits(0x4041424344454647), math.Float64frombits(0x4031323334353637)), []byte{gbComplex8, 0x47, 0x46, 0x45, 0x44, 0x43, 0x42, 0x41, 0x40, gbComplex8, 0x37, 0x36, 0x35, 0x34, 0x33, 0x32, 0x31, 0x40}},
+		{[]byte(""), []byte{gbBytes00}},
+		{[]byte("f"), []byte{gbBytes01, 'f'}},
+		{[]byte("fo"), []byte{gbBytes02, 'f', 'o'}},
+		{[]byte("0123456789abcdefx"), []byte{gbBytes17, '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'x'}},
+		{[]byte("0123456789abcdefxy"), []byte{gbBytes1, 18, '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'x', 'y'}},
+		{[]byte(s256[:255]), append([]byte{gbBytes1, 0xff}, []byte(s256[:255])...)},
+		{[]byte(s256), append([]byte{gbBytes2, 0x00, 0xff}, []byte(s256)...)},
+		{"", []byte{gbString00}},
+		{"f", []byte{gbString01, 'f'}},
+		{"fo", []byte{gbString02, 'f', 'o'}},
+		{"0123456789abcdefx", []byte{gbString17, '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'x'}},
+		{"0123456789abcdefxy", []byte{gbString1, 18, '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'x', 'y'}},
+		{s256[:255], append([]byte{gbString1, 0xff}, []byte(s256[:255])...)},
+		{s256, append([]byte{gbString2, 0x01, 0x00}, []byte(s256)...)},
+		{uint64(0xff), []byte{gbUintP1, 255}},
+		{uint64(0xffff), []byte{gbUintP2, 255, 255}},
+		{uint64(0xffffff), []byte{gbUintP3, 255, 255, 255}},
+		{uint64(0xffffffff), []byte{gbUintP4, 255, 255, 255, 255}},
+		{uint64(0xffffffffff), []byte{gbUintP5, 255, 255, 255, 255, 255}},
+		{uint64(0xffffffffffff), []byte{gbUintP6, 255, 255, 255, 255, 255, 255}},
+		{uint64(0xffffffffffffff), []byte{gbUintP7, 255, 255, 255, 255, 255, 255, 255}},
+		{uint64(0xffffffffffffffff), []byte{gbUintP8, 255, 255, 255, 255, 255, 255, 255, 255}},
+		{int64(math.MinInt64), []byte{gbIntM8, 128, 0, 0, 0, 0, 0, 0, 0}},
+		{-int64(0x100000000000000), []byte{gbIntM7, 0, 0, 0, 0, 0, 0, 0}},
+		{-int64(0x1000000000000), []byte{gbIntM6, 0, 0, 0, 0, 0, 0}},
+		{-int64(0x10000000000), []byte{gbIntM5, 0, 0, 0, 0, 0}},
+		{-int64(0x100000000), []byte{gbIntM4, 0, 0, 0, 0}},
+		{-int64(0x1000000), []byte{gbIntM3, 0, 0, 0}},
+		{-int64(0x10000), []byte{gbIntM2, 0, 0}},
+		{-int64(0x100), []byte{gbIntM1, 0}},
+		{-int64(0xff), []byte{gbIntM1, 1}},
+		{-int64(1), []byte{gbIntM1, 255}},
+		{int64(gbIntMax + 1), []byte{gbIntP1, gbIntMax + 1}},
+		{int64(0xff), []byte{gbIntP1, 255}},
+		{int64(0xffff), []byte{gbIntP2, 255, 255}},
+		{int64(0xffffff), []byte{gbIntP3, 255, 255, 255}},
+		{int64(0xffffffff), []byte{gbIntP4, 255, 255, 255, 255}},
+		{int64(0xffffffffff), []byte{gbIntP5, 255, 255, 255, 255, 255}},
+		{int64(0xffffffffffff), []byte{gbIntP6, 255, 255, 255, 255, 255, 255}},
+		{int64(0xffffffffffffff), []byte{gbIntP7, 255, 255, 255, 255, 255, 255, 255}},
+		{int64(0x7fffffffffffffff), []byte{gbIntP8, 127, 255, 255, 255, 255, 255, 255, 255}},
+		{int64(0), []byte{0 + gbInt0}},
+		{int64(1), []byte{1 + gbInt0}},
+		{int64(2), []byte{2 + gbInt0}},
+		{int64(gbIntMax - 2), "fd"},
+		{int64(gbIntMax - 1), "fe"},
+		{int64(gbIntMax), "ff"},
+	}
+
+	for i, v := range table {
+		g, err := EncodeScalars(v.v)
+		if err != nil {
+			t.Fatal(i, err)
+		}
+
+		var e []byte
+		switch x := v.exp.(type) {
+		case string:
+			e = s2b(x)
+		case []byte:
+			e = x
+		}
+
+		if !bytes.Equal(g, e) {
+			t.Fatalf("%d %v\n|% 02x|\n|% 02x|", i, v.v, g, e)
+		}
+
+		t.Logf("%#v |% 02x|", v.v, g)
+
+		dec, err := DecodeScalars(g)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if g, e := len(dec), 1; g != e {
+			t.Fatalf("%d %d %#v", g, e, dec)
+		}
+
+		if g, ok := dec[0].([]byte); ok {
+			if e := v.v.([]byte); !bytes.Equal(g, e) {
+				t.Fatal(g, e)
+			}
+
+			continue
+		}
+
+		if g, e := dec[0], v.v; g != e {
+			t.Fatal(g, e)
+		}
+	}
+}
+
+func strcmp(a, b string) (r int) {
+	if a < b {
+		return -1
+	}
+
+	if a == b {
+		return 0
+	}
+
+	return 1
+}
+
+func TestCollateScalars(t *testing.T) {
+	// all cases must return -1
+	table := []struct{ x, y []interface{} }{
+		{[]interface{}{}, []interface{}{1}},
+		{[]interface{}{1}, []interface{}{2}},
+		{[]interface{}{1, 2}, []interface{}{2, 3}},
+
+		{[]interface{}{nil}, []interface{}{nil, true}},
+		{[]interface{}{nil}, []interface{}{false}},
+		{[]interface{}{nil}, []interface{}{nil, 1}},
+		{[]interface{}{nil}, []interface{}{1}},
+		{[]interface{}{nil}, []interface{}{nil, uint(1)}},
+		{[]interface{}{nil}, []interface{}{uint(1)}},
+		{[]interface{}{nil}, []interface{}{nil, 3.14}},
+		{[]interface{}{nil}, []interface{}{3.14}},
+		{[]interface{}{nil}, []interface{}{nil, 3.14 + 1i}},
+		{[]interface{}{nil}, []interface{}{3.14 + 1i}},
+		{[]interface{}{nil}, []interface{}{nil, []byte("foo")}},
+		{[]interface{}{nil}, []interface{}{[]byte("foo")}},
+		{[]interface{}{nil}, []interface{}{nil, "foo"}},
+		{[]interface{}{nil}, []interface{}{"foo"}},
+
+		{[]interface{}{false}, []interface{}{false, false}},
+		{[]interface{}{false}, []interface{}{false, true}},
+		{[]interface{}{false}, []interface{}{true}},
+		{[]interface{}{false}, []interface{}{false, 1}},
+		{[]interface{}{false}, []interface{}{1}},
+		{[]interface{}{false}, []interface{}{false, uint(1)}},
+		{[]interface{}{false}, []interface{}{uint(1)}},
+		{[]interface{}{false}, []interface{}{false, 1.5}},
+		{[]interface{}{false}, []interface{}{1.5}},
+		{[]interface{}{false}, []interface{}{false, 1.5 + 3i}},
+		{[]interface{}{false}, []interface{}{1.5 + 3i}},
+		{[]interface{}{false}, []interface{}{false, []byte("foo")}},
+		{[]interface{}{false}, []interface{}{[]byte("foo")}},
+		{[]interface{}{false}, []interface{}{false, "foo"}},
+		{[]interface{}{false}, []interface{}{"foo"}},
+
+		{[]interface{}{1}, []interface{}{1, 2}},
+		{[]interface{}{1}, []interface{}{1, 1}},
+		{[]interface{}{1}, []interface{}{1, uint(2)}},
+		{[]interface{}{1}, []interface{}{uint(2)}},
+		{[]interface{}{1}, []interface{}{1, 1.1}},
+		{[]interface{}{1}, []interface{}{1.1}},
+		{[]interface{}{1}, []interface{}{1, 1.1 + 2i}},
+		{[]interface{}{1}, []interface{}{1.1 + 2i}},
+		{[]interface{}{1}, []interface{}{1, []byte("foo")}},
+		{[]interface{}{1}, []interface{}{[]byte("foo")}},
+		{[]interface{}{1}, []interface{}{1, "foo"}},
+		{[]interface{}{1}, []interface{}{"foo"}},
+
+		{[]interface{}{uint(1)}, []interface{}{uint(1), uint(1)}},
+		{[]interface{}{uint(1)}, []interface{}{uint(2)}},
+		{[]interface{}{uint(1)}, []interface{}{uint(1), 2.}},
+		{[]interface{}{uint(1)}, []interface{}{2.}},
+		{[]interface{}{uint(1)}, []interface{}{uint(1), 2. + 0i}},
+		{[]interface{}{uint(1)}, []interface{}{2. + 0i}},
+		{[]interface{}{uint(1)}, []interface{}{uint(1), []byte("foo")}},
+		{[]interface{}{uint(1)}, []interface{}{[]byte("foo")}},
+		{[]interface{}{uint(1)}, []interface{}{uint(1), "foo"}},
+		{[]interface{}{uint(1)}, []interface{}{"foo"}},
+
+		{[]interface{}{1.}, []interface{}{1., 1}},
+		{[]interface{}{1.}, []interface{}{2}},
+		{[]interface{}{1.}, []interface{}{1., uint(1)}},
+		{[]interface{}{1.}, []interface{}{uint(2)}},
+		{[]interface{}{1.}, []interface{}{1., 1.}},
+		{[]interface{}{1.}, []interface{}{1.1}},
+		{[]interface{}{1.}, []interface{}{1., []byte("foo")}},
+		{[]interface{}{1.}, []interface{}{[]byte("foo")}},
+		{[]interface{}{1.}, []interface{}{1., "foo"}},
+		{[]interface{}{1.}, []interface{}{"foo"}},
+
+		{[]interface{}{1 + 2i}, []interface{}{1 + 2i, 1}},
+		{[]interface{}{1 + 2i}, []interface{}{2}},
+		{[]interface{}{1 + 2i}, []interface{}{1 + 2i, uint(1)}},
+		{[]interface{}{1 + 2i}, []interface{}{uint(2)}},
+		{[]interface{}{1 + 2i}, []interface{}{1 + 2i, 1.1}},
+		{[]interface{}{1 + 2i}, []interface{}{1.1}},
+		{[]interface{}{1 + 2i}, []interface{}{1 + 2i, []byte("foo")}},
+		{[]interface{}{1 + 2i}, []interface{}{[]byte("foo")}},
+		{[]interface{}{1 + 2i}, []interface{}{1 + 2i, "foo"}},
+		{[]interface{}{1 + 2i}, []interface{}{"foo"}},
+
+		{[]interface{}{[]byte("bar")}, []interface{}{[]byte("bar"), []byte("bar")}},
+		{[]interface{}{[]byte("bar")}, []interface{}{[]byte("foo")}},
+		{[]interface{}{[]byte("bar")}, []interface{}{[]byte("c")}},
+		{[]interface{}{[]byte("bar")}, []interface{}{[]byte("bas")}},
+		{[]interface{}{[]byte("bar")}, []interface{}{[]byte("bara")}},
+
+		{[]interface{}{[]byte("bar")}, []interface{}{"bap"}},
+		{[]interface{}{[]byte("bar")}, []interface{}{"bar"}},
+		{[]interface{}{[]byte("bar")}, []interface{}{"bas"}},
+
+		{[]interface{}{"bar"}, []interface{}{"bar", "bar"}},
+		{[]interface{}{"bar"}, []interface{}{"foo"}},
+		{[]interface{}{"bar"}, []interface{}{"c"}},
+		{[]interface{}{"bar"}, []interface{}{"bas"}},
+		{[]interface{}{"bar"}, []interface{}{"bara"}},
+
+		{[]interface{}{1 + 2i}, []interface{}{1 + 3i}},
+		{[]interface{}{int64(math.MaxInt64)}, []interface{}{uint64(math.MaxInt64 + 1)}},
+		{[]interface{}{int8(1)}, []interface{}{int16(2)}},
+		{[]interface{}{int32(1)}, []interface{}{uint8(2)}},
+		{[]interface{}{uint16(1)}, []interface{}{uint32(2)}},
+		{[]interface{}{float32(1)}, []interface{}{complex(float32(2), 0)}},
+
+		// resolved bugs
+		{[]interface{}{"Customer"}, []interface{}{"Date"}},
+		{[]interface{}{"Customer"}, []interface{}{"Items", 1, "Quantity"}},
+	}
+
+	more := []interface{}{42, nil, 1, uint(2), 3.0, 4 + 5i, "..."}
+
+	collate := func(x, y []interface{}, strCollate func(string, string) int) (r int) {
+		var err error
+		r, err = Collate(x, y, strCollate)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		return
+	}
+
+	for _, scf := range []func(string, string) int{nil, strcmp} {
+		for _, prefix := range more {
+			for i, test := range table {
+				var x, y []interface{}
+				if prefix != 42 {
+					x = append(x, prefix)
+					y = append(y, prefix)
+				}
+				x = append(x, test.x...)
+				y = append(y, test.y...)
+
+				// cmp(x, y) == -1
+				if g, e := collate(x, y, scf), -1; g != e {
+					t.Fatal(i, g, e, x, y)
+				}
+
+				// cmp(y, x) == 1
+				if g, e := collate(y, x, scf), 1; g != e {
+					t.Fatal(i, g, e, y, x)
+				}
+
+				src := x
+				for ix := len(src) - 1; ix > 0; ix-- {
+					if g, e := collate(src[:ix], src[:ix], scf), 0; g != e {
+						t.Fatal(ix, g, e)
+					}
+
+					if g, e := collate(src[:ix], src, scf), -1; g != e {
+						t.Fatal(ix, g, e)
+					}
+
+				}
+
+				src = y
+				for ix := len(src) - 1; ix > 0; ix-- {
+					if g, e := collate(src[:ix], src[:ix], scf), 0; g != e {
+						t.Fatal(ix, g, e)
+					}
+
+					if g, e := collate(src[:ix], src, scf), -1; g != e {
+						t.Fatal(ix, g, e)
+					}
+
+				}
+			}
+		}
+	}
+}
+
+func TestEncodingBug(t *testing.T) {
+	bits := uint64(0)
+	for i := 0; i <= 64; i++ {
+		encoded, err := EncodeScalars(math.Float64frombits(bits))
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		t.Logf("bits %016x, enc |% x|", bits, encoded)
+		decoded, err := DecodeScalars(encoded)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if g, e := len(decoded), 1; g != e {
+			t.Fatal(g, e)
+		}
+
+		f, ok := decoded[0].(float64)
+		if !ok {
+			t.Fatal(err)
+		}
+
+		if g, e := math.Float64bits(f), bits; g != e {
+			t.Fatal(err)
+		}
+
+		t.Log(f)
+
+		bits >>= 1
+		bits |= 1 << 63
+	}
+}

Godeps/_workspace/src/github.com/cznic/exp/lldb/lldb.go

@@ -0,0 +1,155 @@
+// Copyright 2014 The lldb Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package lldb (WIP) implements a low level database engine. The database
+// model used could be considered a specific implementation of some small(est)
+// intersection of models listed in [1]. As a settled term is lacking, it'll be
+// called here a 'Virtual memory model' (VMM).
+//
+// Experimental release notes
+//
+// This is an experimental release. Don't open a DB from two applications or
+// two instances of an application - it will get corrupted (no file locking is
+// implemented and this task is delegated to lldb's clients).
+//
+// WARNING: THE LLDB API IS SUBJECT TO CHANGE.
+//
+// Filers
+//
+// A Filer is an abstraction of storage. A Filer may be a part of some process'
+// virtual address space, an OS file, a networked, remote file etc. Persistence
+// of the storage is optional, opaque to VMM and it is specific to a concrete
+// Filer implementation.
+//
+// Space management
+//
+// Mechanism to allocate, reallocate (resize), deallocate (and later reclaim
+// the unused) contiguous parts of a Filer, called blocks.  Blocks are
+// identified and referred to by a handle, an int64.
+//
+// BTrees
+//
+// In addition to the VMM like services, lldb provides volatile and
+// non-volatile BTrees. Keys and values of a BTree are limited in size to 64kB
+// each (a bit more actually). Support for larger keys/values, if desired, can
+// be built atop a BTree to certain limits.
+//
+// Handles vs pointers
+//
+// A handle is the abstracted storage counterpart of a memory address.  There
+// is one fundamental difference, though. Resizing a block never results in a
+// change to the handle which refers to the resized block, so a handle is more
+// akin to an unique numeric id/key. Yet it shares one property of pointers -
+// handles can be associated again with blocks after the original handle block
+// was deallocated. In other words, a handle uniqueness domain is the state of
+// the database and is not something comparable to e.g. an ever growing
+// numbering sequence.
+//
+// Also, as with memory pointers, dangling handles can be created and blocks
+// overwritten when such handles are used. Using a zero handle to refer to a
+// block will not panic; however, the resulting error is effectively the same
+// exceptional situation as dereferencing a nil pointer.
+//
+// Blocks
+//
+// Allocated/used blocks, are limited in size to only a little bit more than
+// 64kB.  Bigger semantic entities/structures must be built in lldb's client
+// code.  The content of a block has no semantics attached, it's only a fully
+// opaque `[]byte`.
+//
+// Scalars
+//
+// Use of "scalars" applies to EncodeScalars, DecodeScalars and Collate. Those
+// first two "to bytes" and "from bytes" functions are suggested for handling
+// multi-valued Allocator content items and/or keys/values of BTrees (using
+// Collate for keys). Types called "scalar" are:
+//
+//	nil (the typeless one)
+//	bool
+//	all integral types: [u]int8, [u]int16, [u]int32, [u]int, [u]int64
+//	all floating point types: float32, float64
+//	all complex types: complex64, complex128
+//	[]byte (64kB max)
+//	string (64kb max)
+//
+// Specific implementations
+//
+// Included are concrete implementations of some of the VMM interfaces included
+// to ease serving simple client code or for testing and possibly as an
+// example.  More details in the documentation of such implementations.
+//
+//  [1]: http://en.wikipedia.org/wiki/Database_model
+package lldb
+
+const (
+	fltSz            = 0x70 // size of the FLT
+	maxShort         = 251
+	maxRq            = 65787
+	maxFLTRq         = 4112
+	maxHandle        = 1<<56 - 1
+	atomLen          = 16
+	tagUsedLong      = 0xfc
+	tagUsedRelocated = 0xfd
+	tagFreeShort     = 0xfe
+	tagFreeLong      = 0xff
+	tagNotCompressed = 0
+	tagCompressed    = 1
+)
+
+// Content size n -> blocksize in atoms.
+func n2atoms(n int) int {
+	if n > maxShort {
+		n += 2
+	}
+	return (n+1)/16 + 1
+}
+
+// Content size n -> number of padding zeros.
+func n2padding(n int) int {
+	if n > maxShort {
+		n += 2
+	}
+	return 15 - (n+1)&15
+}
+
+// Handle <-> offset
+func h2off(h int64) int64   { return (h + 6) * 16 }
+func off2h(off int64) int64 { return off/16 - 6 }
+
+// Get a 7B int64 from b
+func b2h(b []byte) (h int64) {
+	for _, v := range b[:7] {
+		h = h<<8 | int64(v)
+	}
+	return
+}
+
+// Put a 7B int64 into b
+func h2b(b []byte, h int64) []byte {
+	for i := range b[:7] {
+		b[i], h = byte(h>>48), h<<8
+	}
+	return b
+}
+
+// Content length N (must be in [252, 65787]) to long used block M field.
+func n2m(n int) (m int) {
+	return n % 0x10000
+}
+
+// Long used block M (must be in [0, 65535]) field to content length N.
+func m2n(m int) (n int) {
+	if m <= maxShort {
+		m += 0x10000
+	}
+	return m
+}
+
+func bpack(a []byte) []byte {
+	if cap(a) > len(a) {
+		return append([]byte(nil), a...)
+	}
+
+	return a
+}

Godeps/_workspace/src/github.com/cznic/exp/lldb/lldb_test.go

@@ -0,0 +1,217 @@
+// Copyright 2014 The lldb Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package lldb
+
+import (
+	"bytes"
+	"encoding/hex"
+	"fmt"
+	"path"
+	"runtime"
+	"strings"
+	"testing"
+)
+
+var dbg = func(s string, va ...interface{}) {
+	_, fn, fl, _ := runtime.Caller(1)
+	fmt.Printf("%s:%d: ", path.Base(fn), fl)
+	fmt.Printf(s+"\n", va...)
+}
+
+func use(...interface{}) {}
+
+func TestN2Atoms(t *testing.T) {
+	tab := []struct{ n, a int }{
+		{0, 1},
+		{1, 1},
+		{2, 1},
+		{3, 1},
+		{4, 1},
+		{5, 1},
+		{6, 1},
+		{7, 1},
+		{8, 1},
+		{9, 1},
+		{10, 1},
+		{11, 1},
+		{12, 1},
+		{13, 1},
+		{14, 1},
+
+		{15, 2},
+		{16, 2},
+		{17, 2},
+		{18, 2},
+		{19, 2},
+		{20, 2},
+		{21, 2},
+		{22, 2},
+		{23, 2},
+		{24, 2},
+		{25, 2},
+		{26, 2},
+		{27, 2},
+		{28, 2},
+		{29, 2},
+		{30, 2},
+
+		{31, 3},
+
+		{252, 16},
+		{253, 17},
+		{254, 17},
+		{255, 17},
+		{256, 17},
+		{257, 17},
+		{258, 17},
+		{259, 17},
+		{260, 17},
+		{261, 17},
+		{262, 17},
+		{263, 17},
+		{264, 17},
+		{265, 17},
+		{266, 17},
+		{267, 17},
+		{268, 17},
+		{269, 18},
+		{65532, 4096},
+		{65533, 4097},
+		{65787, 4112},
+	}
+
+	for i, test := range tab {
+		if g, e := n2atoms(test.n), test.a; g != e {
+			t.Errorf("(%d) %d %d %d", i, test.n, g, e)
+		}
+	}
+}
+
+func TestN2Padding(t *testing.T) {
+	tab := []struct{ n, p int }{
+		{0, 14},
+		{1, 13},
+		{2, 12},
+		{3, 11},
+		{4, 10},
+		{5, 9},
+		{6, 8},
+		{7, 7},
+		{8, 6},
+		{9, 5},
+		{10, 4},
+		{11, 3},
+		{12, 2},
+		{13, 1},
+		{14, 0},
+
+		{15, 15},
+		{16, 14},
+		{17, 13},
+		{18, 12},
+		{19, 11},
+		{20, 10},
+		{21, 9},
+		{22, 8},
+		{23, 7},
+		{24, 6},
+		{25, 5},
+		{26, 4},
+		{27, 3},
+		{28, 2},
+		{29, 1},
+		{30, 0},
+
+		{31, 15},
+
+		{252, 0},
+		{253, 15},
+		{254, 14},
+		{255, 13},
+		{256, 12},
+		{257, 11},
+		{258, 10},
+		{259, 9},
+		{260, 8},
+		{261, 7},
+		{262, 6},
+		{263, 5},
+		{264, 4},
+		{265, 3},
+		{266, 2},
+		{267, 1},
+		{268, 0},
+		{269, 15},
+	}
+
+	for i, test := range tab {
+		if g, e := n2padding(test.n), test.p; g != e {
+			t.Errorf("(%d) %d %d %d", i, test.n, g, e)
+		}
+	}
+}
+
+func TestH2Off(t *testing.T) {
+	tab := []struct{ h, off int64 }{
+		{-1, fltSz - 32},
+		{0, fltSz - 16},
+		{1, fltSz + 0},
+		{2, fltSz + 16},
+		{3, fltSz + 32},
+	}
+
+	for i, test := range tab {
+		if g, e := h2off(test.h), test.off; g != e {
+			t.Error("h2off", i, g, e)
+		}
+		if g, e := off2h(test.off), test.h; g != e {
+			t.Error("off2h", i, g, e)
+		}
+	}
+}
+
+func TestB2H(t *testing.T) {
+	tab := []struct {
+		b []byte
+		h int64
+	}{
+		{[]byte{0, 0, 0, 0, 0, 0, 0}, 0},
+		{[]byte{0, 0, 0, 0, 0, 0, 1}, 1},
+		{[]byte{0, 0, 0, 0, 0, 0, 1, 2}, 1},
+		{[]byte{0, 0, 0, 0, 0, 0x32, 0x10}, 0x3210},
+		{[]byte{0, 0, 0, 0, 0x54, 0x32, 0x10}, 0x543210},
+		{[]byte{0, 0, 0, 0x76, 0x54, 0x32, 0x10}, 0x76543210},
+		{[]byte{0, 0, 0x98, 0x76, 0x54, 0x32, 0x10}, 0x9876543210},
+		{[]byte{0, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10}, 0xba9876543210},
+		{[]byte{0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10}, 0xdcba9876543210},
+	}
+
+	for i, test := range tab {
+		if g, e := b2h(test.b), test.h; g != e {
+			t.Errorf("b2h: %d %#8x %#8x", i, g, e)
+		}
+		var g [7]byte
+		h2b(g[:], test.h)
+		if e := test.b; !bytes.Equal(g[:], e[:7]) {
+			t.Errorf("b2h: %d g: % 0x e: % 0x", i, g, e)
+		}
+	}
+}
+
+func s2b(s string) []byte {
+	if s == "" {
+		return nil
+	}
+
+	s = strings.Replace(s, " ", "", -1)
+	if n := len(s) & 1; n != 0 {
+		panic(n)
+	}
+	b, err := hex.DecodeString(s)
+	if err != nil {
+		panic(err)
+	}
+	return b
+}

Godeps/_workspace/src/github.com/cznic/exp/lldb/memfiler.go

@@ -0,0 +1,344 @@
+// Copyright 2014 The lldb Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// A memory-only implementation of Filer.
+
+/*
+
+pgBits: 8
+BenchmarkMemFilerWrSeq	  100000	     19430 ns/op	1646.93 MB/s
+BenchmarkMemFilerRdSeq	  100000	     17390 ns/op	1840.13 MB/s
+BenchmarkMemFilerWrRand	 1000000	      1903 ns/op	 133.94 MB/s
+BenchmarkMemFilerRdRand	 1000000	      1153 ns/op	 221.16 MB/s
+
+pgBits: 9
+BenchmarkMemFilerWrSeq	  100000	     16195 ns/op	1975.80 MB/s
+BenchmarkMemFilerRdSeq	  200000	     13011 ns/op	2459.39 MB/s
+BenchmarkMemFilerWrRand	 1000000	      2248 ns/op	 227.28 MB/s
+BenchmarkMemFilerRdRand	 1000000	      1177 ns/op	 433.94 MB/s
+
+pgBits: 10
+BenchmarkMemFilerWrSeq	  100000	     16169 ns/op	1979.04 MB/s
+BenchmarkMemFilerRdSeq	  200000	     12673 ns/op	2524.91 MB/s
+BenchmarkMemFilerWrRand	 1000000	      5550 ns/op	 184.30 MB/s
+BenchmarkMemFilerRdRand	 1000000	      1699 ns/op	 601.79 MB/s
+
+pgBits: 11
+BenchmarkMemFilerWrSeq	  100000	     13449 ns/op	2379.31 MB/s
+BenchmarkMemFilerRdSeq	  200000	     12058 ns/op	2653.80 MB/s
+BenchmarkMemFilerWrRand	  500000	      4335 ns/op	 471.47 MB/s
+BenchmarkMemFilerRdRand	 1000000	      2843 ns/op	 719.47 MB/s
+
+pgBits: 12
+BenchmarkMemFilerWrSeq	  200000	     11976 ns/op	2672.00 MB/s
+BenchmarkMemFilerRdSeq	  200000	     12255 ns/op	2611.06 MB/s
+BenchmarkMemFilerWrRand	  200000	      8058 ns/op	 507.14 MB/s
+BenchmarkMemFilerRdRand	  500000	      4365 ns/op	 936.15 MB/s
+
+pgBits: 13
+BenchmarkMemFilerWrSeq	  200000	     10852 ns/op	2948.69 MB/s
+BenchmarkMemFilerRdSeq	  200000	     11561 ns/op	2767.77 MB/s
+BenchmarkMemFilerWrRand	  200000	      9748 ns/op	 840.15 MB/s
+BenchmarkMemFilerRdRand	  500000	      7236 ns/op	1131.59 MB/s
+
+pgBits: 14
+BenchmarkMemFilerWrSeq	  200000	     10328 ns/op	3098.12 MB/s
+BenchmarkMemFilerRdSeq	  200000	     11292 ns/op	2833.66 MB/s
+BenchmarkMemFilerWrRand	  100000	     16768 ns/op	 978.75 MB/s
+BenchmarkMemFilerRdRand	  200000	     13033 ns/op	1258.43 MB/s
+
+pgBits: 15
+BenchmarkMemFilerWrSeq	  200000	     10309 ns/op	3103.93 MB/s
+BenchmarkMemFilerRdSeq	  200000	     11126 ns/op	2876.12 MB/s
+BenchmarkMemFilerWrRand	   50000	     31985 ns/op	1021.74 MB/s
+BenchmarkMemFilerRdRand	  100000	     25217 ns/op	1297.65 MB/s
+
+pgBits: 16
+BenchmarkMemFilerWrSeq	  200000	     10324 ns/op	3099.45 MB/s
+BenchmarkMemFilerRdSeq	  200000	     11201 ns/op	2856.80 MB/s
+BenchmarkMemFilerWrRand	   20000	     55226 ns/op	1184.76 MB/s
+BenchmarkMemFilerRdRand	   50000	     48316 ns/op	1355.16 MB/s
+
+pgBits: 17
+BenchmarkMemFilerWrSeq	  200000	     10377 ns/op	3083.53 MB/s
+BenchmarkMemFilerRdSeq	  200000	     11018 ns/op	2904.18 MB/s
+BenchmarkMemFilerWrRand	   10000	    143425 ns/op	 913.12 MB/s
+BenchmarkMemFilerRdRand	   20000	     95267 ns/op	1376.99 MB/s
+
+pgBits: 18
+BenchmarkMemFilerWrSeq	  200000	     10312 ns/op	3102.96 MB/s
+BenchmarkMemFilerRdSeq	  200000	     11069 ns/op	2890.84 MB/s
+BenchmarkMemFilerWrRand	    5000	    280910 ns/op	 934.14 MB/s
+BenchmarkMemFilerRdRand	   10000	    188500 ns/op	1388.17 MB/s
+
+*/
+
+package lldb
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+
+	"github.com/coreos/rocket/Godeps/_workspace/src/github.com/cznic/fileutil"
+	"github.com/coreos/rocket/Godeps/_workspace/src/github.com/cznic/mathutil"
+)
+
+const (
+	pgBits = 16
+	pgSize = 1 << pgBits
+	pgMask = pgSize - 1
+)
+
+var _ Filer = &MemFiler{} // Ensure MemFiler is a Filer.
+
+type memFilerMap map[int64]*[pgSize]byte
+
+// MemFiler is a memory backed Filer. It implements BeginUpdate, EndUpdate and
+// Rollback as no-ops. MemFiler is not automatically persistent, but it has
+// ReadFrom and WriteTo methods.
+type MemFiler struct {
+	m    memFilerMap
+	nest int
+	size int64
+}
+
+// NewMemFiler returns a new MemFiler.
+func NewMemFiler() *MemFiler {
+	return &MemFiler{m: memFilerMap{}}
+}
+
+// BeginUpdate implements Filer.
+func (f *MemFiler) BeginUpdate() error {
+	f.nest++
+	return nil
+}
+
+// Close implements Filer.
+func (f *MemFiler) Close() (err error) {
+	if f.nest != 0 {
+		return &ErrPERM{(f.Name() + ":Close")}
+	}
+
+	return
+}
+
+// EndUpdate implements Filer.
+func (f *MemFiler) EndUpdate() (err error) {
+	if f.nest == 0 {
+		return &ErrPERM{(f.Name() + ": EndUpdate")}
+	}
+
+	f.nest--
+	return
+}
+
+// Name implements Filer.
+func (f *MemFiler) Name() string {
+	return fmt.Sprintf("%p.memfiler", f)
+}
+
+// PunchHole implements Filer.
+func (f *MemFiler) PunchHole(off, size int64) (err error) {
+	if off < 0 {
+		return &ErrINVAL{f.Name() + ": PunchHole off", off}
+	}
+
+	if size < 0 || off+size > f.size {
+		return &ErrINVAL{f.Name() + ": PunchHole size", size}
+	}
+
+	first := off >> pgBits
+	if off&pgMask != 0 {
+		first++
+	}
+	off += size - 1
+	last := off >> pgBits
+	if off&pgMask != 0 {
+		last--
+	}
+	if limit := f.size >> pgBits; last > limit {
+		last = limit
+	}
+	for pg := first; pg <= last; pg++ {
+		delete(f.m, pg)
+	}
+	return
+}
+
+var zeroPage [pgSize]byte
+
+// ReadAt implements Filer.
+func (f *MemFiler) ReadAt(b []byte, off int64) (n int, err error) {
+	avail := f.size - off
+	pgI := off >> pgBits
+	pgO := int(off & pgMask)
+	rem := len(b)
+	if int64(rem) >= avail {
+		rem = int(avail)
+		err = io.EOF
+	}
+	for rem != 0 && avail > 0 {
+		pg := f.m[pgI]
+		if pg == nil {
+			pg = &zeroPage
+		}
+		nc := copy(b[:mathutil.Min(rem, pgSize)], pg[pgO:])
+		pgI++
+		pgO = 0
+		rem -= nc
+		n += nc
+		b = b[nc:]
+	}
+	return
+}
+
+// ReadFrom is a helper to populate MemFiler's content from r.  'n' reports the
+// number of bytes read from 'r'.
+func (f *MemFiler) ReadFrom(r io.Reader) (n int64, err error) {
+	if err = f.Truncate(0); err != nil {
+		return
+	}
+
+	var (
+		b   [pgSize]byte
+		rn  int
+		off int64
+	)
+
+	var rerr error
+	for rerr == nil {
+		if rn, rerr = r.Read(b[:]); rn != 0 {
+			f.WriteAt(b[:rn], off)
+			off += int64(rn)
+			n += int64(rn)
+		}
+	}
+	if !fileutil.IsEOF(rerr) {
+		err = rerr
+	}
+	return
+}
+
+// Rollback implements Filer.
+func (f *MemFiler) Rollback() (err error) { return }
+
+// Size implements Filer.
+func (f *MemFiler) Size() (int64, error) {
+	return f.size, nil
+}
+
+// Sync implements Filer.
+func (f *MemFiler) Sync() error {
+	return nil
+}
+
+// Truncate implements Filer.
+func (f *MemFiler) Truncate(size int64) (err error) {
+	switch {
+	case size < 0:
+		return &ErrINVAL{"Truncate size", size}
+	case size == 0:
+		f.m = memFilerMap{}
+		f.size = 0
+		return
+	}
+
+	first := size >> pgBits
+	if size&pgMask != 0 {
+		first++
+	}
+	last := f.size >> pgBits
+	if f.size&pgMask != 0 {
+		last++
+	}
+	for ; first < last; first++ {
+		delete(f.m, first)
+	}
+
+	f.size = size
+	return
+}
+
+// WriteAt implements Filer.
+func (f *MemFiler) WriteAt(b []byte, off int64) (n int, err error) {
+	pgI := off >> pgBits
+	pgO := int(off & pgMask)
+	n = len(b)
+	rem := n
+	var nc int
+	for rem != 0 {
+		if pgO == 0 && rem >= pgSize && bytes.Equal(b[:pgSize], zeroPage[:]) {
+			delete(f.m, pgI)
+			nc = pgSize
+		} else {
+			pg := f.m[pgI]
+			if pg == nil {
+				pg = new([pgSize]byte)
+				f.m[pgI] = pg
+			}
+			nc = copy((*pg)[pgO:], b)
+		}
+		pgI++
+		pgO = 0
+		rem -= nc
+		b = b[nc:]
+	}
+	f.size = mathutil.MaxInt64(f.size, off+int64(n))
+	return
+}
+
+// WriteTo is a helper to copy/persist MemFiler's content to w.  If w is also
+// an io.WriterAt then WriteTo may attempt to _not_ write any big, for some
+// value of big, runs of zeros, i.e. it will attempt to punch holes, where
+// possible, in `w` if that happens to be a freshly created or to zero length
+// truncated OS file.  'n' reports the number of bytes written to 'w'.
+func (f *MemFiler) WriteTo(w io.Writer) (n int64, err error) {
+	var (
+		b      [pgSize]byte
+		wn, rn int
+		off    int64
+		rerr   error
+	)
+
+	if wa, ok := w.(io.WriterAt); ok {
+		lastPgI := f.size >> pgBits
+		for pgI := int64(0); pgI <= lastPgI; pgI++ {
+			sz := pgSize
+			if pgI == lastPgI {
+				sz = int(f.size & pgMask)
+			}
+			pg := f.m[pgI]
+			if pg != nil {
+				wn, err = wa.WriteAt(pg[:sz], off)
+				if err != nil {
+					return
+				}
+
+				n += int64(wn)
+				off += int64(sz)
+				if wn != sz {
+					return n, io.ErrShortWrite
+				}
+			}
+		}
+		return
+	}
+
+	var werr error
+	for rerr == nil {
+		if rn, rerr = f.ReadAt(b[:], off); rn != 0 {
+			off += int64(rn)
+			if wn, werr = w.Write(b[:rn]); werr != nil {
+				return n, werr
+			}
+
+			n += int64(wn)
+		}
+	}
+	if !fileutil.IsEOF(rerr) {
+		err = rerr
+	}
+	return
+}

Godeps/_workspace/src/github.com/cznic/exp/lldb/memfiler_test.go

@@ -0,0 +1,132 @@
+// Copyright 2014 The lldb Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package lldb
+
+import (
+	"bytes"
+	"math/rand"
+	"testing"
+)
+
+// Test automatic page releasing (hole punching) of zero pages
+func TestMemFilerWriteAt(t *testing.T) {
+	f := NewMemFiler()
+
+	// Add page index 0
+	if _, err := f.WriteAt([]byte{1}, 0); err != nil {
+		t.Fatal(err)
+	}
+
+	if g, e := len(f.m), 1; g != e {
+		t.Fatal(g, e)
+	}
+
+	// Add page index 1
+	if _, err := f.WriteAt([]byte{2}, pgSize); err != nil {
+		t.Fatal(err)
+	}
+
+	if g, e := len(f.m), 2; g != e {
+		t.Fatal(g, e)
+	}
+
+	// Add page index 2
+	if _, err := f.WriteAt([]byte{3}, 2*pgSize); err != nil {
+		t.Fatal(err)
+	}
+
+	if g, e := len(f.m), 3; g != e {
+		t.Fatal(g, e)
+	}
+
+	// Remove page index 1
+	if _, err := f.WriteAt(make([]byte, 2*pgSize), pgSize/2); err != nil {
+		t.Fatal(err)
+	}
+
+	if g, e := len(f.m), 2; g != e {
+		t.Logf("%#v", f.m)
+		t.Fatal(g, e)
+	}
+
+	if err := f.Truncate(1); err != nil {
+		t.Fatal(err)
+	}
+
+	if g, e := len(f.m), 1; g != e {
+		t.Logf("%#v", f.m)
+		t.Fatal(g, e)
+	}
+
+	if err := f.Truncate(0); err != nil {
+		t.Fatal(err)
+	}
+
+	if g, e := len(f.m), 0; g != e {
+		t.Logf("%#v", f.m)
+		t.Fatal(g, e)
+	}
+}
+
+func TestMemFilerWriteTo(t *testing.T) {
+	const max = 1e5
+	var b [max]byte
+	rng := rand.New(rand.NewSource(42))
+	for sz := 0; sz < 1e5; sz += 2053 {
+		for i := range b[:sz] {
+			b[i] = byte(rng.Int())
+		}
+		f := NewMemFiler()
+		if n, err := f.WriteAt(b[:sz], 0); n != sz || err != nil {
+			t.Fatal(n, err)
+		}
+
+		var buf bytes.Buffer
+		if n, err := f.WriteTo(&buf); n != int64(sz) || err != nil {
+			t.Fatal(n, err)
+		}
+
+		if !bytes.Equal(b[:sz], buf.Bytes()) {
+			t.Fatal("content differs")
+		}
+	}
+}
+
+func TestMemFilerReadFromWriteTo(t *testing.T) {
+	const (
+		sz   = 1e2 * pgSize
+		hole = 1e1 * pgSize
+	)
+	rng := rand.New(rand.NewSource(42))
+	data := make([]byte, sz)
+	for i := range data {
+		data[i] = byte(rng.Int())
+	}
+	f := NewMemFiler()
+	buf := bytes.NewBuffer(data)
+	if n, err := f.ReadFrom(buf); n != int64(len(data)) || err != nil {
+		t.Fatal(n, err)
+	}
+
+	buf = bytes.NewBuffer(nil)
+	if n, err := f.WriteTo(buf); n != int64(len(data)) || err != nil {
+		t.Fatal(n, err)
+	}
+
+	rd := buf.Bytes()
+	if !bytes.Equal(data, rd) {
+		t.Fatal("corrupted data")
+	}
+
+	n0 := len(f.m)
+	data = make([]byte, hole)
+	f.WriteAt(data, sz/2)
+	n := len(f.m)
+	t.Log(n0, n)
+	d := n0 - n
+	if d*pgSize < hole-2 || d*pgSize > hole {
+		t.Fatal(n0, n, d)
+	}
+}

Godeps/_workspace/src/github.com/cznic/exp/lldb/osfiler.go

@@ -0,0 +1,130 @@
+// Copyright 2014 The lldb Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package lldb
+
+import (
+	"io"
+	"os"
+
+	"github.com/coreos/rocket/Godeps/_workspace/src/github.com/cznic/mathutil"
+)
+
+var _ Filer = (*OSFiler)(nil)
+
+// OSFile is an os.File like minimal set of methods allowing to construct a
+// Filer.
+type OSFile interface {
+	Name() string
+	Stat() (fi os.FileInfo, err error)
+	Sync() (err error)
+	Truncate(size int64) (err error)
+	io.Closer
+	io.Reader
+	io.ReaderAt
+	io.Seeker
+	io.Writer
+	io.WriterAt
+}
+
+// OSFiler is like a SimpleFileFiler but based on an OSFile.
+type OSFiler struct {
+	f    OSFile
+	nest int
+	size int64 // not set if < 0
+}
+
+// NewOSFiler returns a Filer from an OSFile. This Filer is like the
+// SimpleFileFiler, it does not implement the transaction related methods.
+func NewOSFiler(f OSFile) (r *OSFiler) {
+	return &OSFiler{
+		f:    f,
+		size: -1,
+	}
+}
+
+// BeginUpdate implements Filer.
+func (f *OSFiler) BeginUpdate() (err error) {
+	f.nest++
+	return nil
+}
+
+// Close implements Filer.
+func (f *OSFiler) Close() (err error) {
+	if f.nest != 0 {
+		return &ErrPERM{(f.Name() + ":Close")}
+	}
+
+	return f.f.Close()
+}
+
+// EndUpdate implements Filer.
+func (f *OSFiler) EndUpdate() (err error) {
+	if f.nest == 0 {
+		return &ErrPERM{(f.Name() + ":EndUpdate")}
+	}
+
+	f.nest--
+	return
+}
+
+// Name implements Filer.
+func (f *OSFiler) Name() string {
+	return f.f.Name()
+}
+
+// PunchHole implements Filer.
+func (f *OSFiler) PunchHole(off, size int64) (err error) {
+	return
+}
+
+// ReadAt implements Filer.
+func (f *OSFiler) ReadAt(b []byte, off int64) (n int, err error) {
+	return f.f.ReadAt(b, off)
+}
+
+// Rollback implements Filer.
+func (f *OSFiler) Rollback() (err error) { return }
+
+// Size implements Filer.
+func (f *OSFiler) Size() (n int64, err error) {
+	if f.size < 0 { // boot
+		fi, err := f.f.Stat()
+		if err != nil {
+			return 0, err
+		}
+
+		f.size = fi.Size()
+	}
+	return f.size, nil
+}
+
+// Sync implements Filer.
+func (f *OSFiler) Sync() (err error) {
+	return f.f.Sync()
+}
+
+// Truncate implements Filer.
+func (f *OSFiler) Truncate(size int64) (err error) {
+	if size < 0 {
+		return &ErrINVAL{"Truncate size", size}
+	}
+
+	f.size = size
+	return f.f.Truncate(size)
+}
+
+// WriteAt implements Filer.
+func (f *OSFiler) WriteAt(b []byte, off int64) (n int, err error) {
+	if f.size < 0 { // boot
+		fi, err := os.Stat(f.f.Name())
+		if err != nil {
+			return 0, err
+		}
+
+		f.size = fi.Size()
+	}
+	f.size = mathutil.MaxInt64(f.size, int64(len(b))+off)
+	return f.f.WriteAt(b, off)
+}

Godeps/_workspace/src/github.com/cznic/exp/lldb/simplefilefiler.go

@@ -0,0 +1,123 @@
+// Copyright 2014 The lldb Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// A basic os.File backed Filer.
+
+package lldb
+
+import (
+	"os"
+
+	"github.com/coreos/rocket/Godeps/_workspace/src/github.com/cznic/fileutil"
+	"github.com/coreos/rocket/Godeps/_workspace/src/github.com/cznic/mathutil"
+)
+
+var _ Filer = &SimpleFileFiler{} // Ensure SimpleFileFiler is a Filer.
+
+// SimpleFileFiler is an os.File backed Filer intended for use where structural
+// consistency can be reached by other means (SimpleFileFiler is for example
+// wrapped in eg. an RollbackFiler or ACIDFiler0) or where persistence is not
+// required (temporary/working data sets).
+//
+// SimpleFileFiler is the most simple os.File backed Filer implementation as it
+// does not really implement BeginUpdate and EndUpdate/Rollback in any way
+// which would protect the structural integrity of data. If misused e.g. as a
+// real database storage w/o other measures, it can easily cause data loss
+// when, for example, a power outage occurs or the updating process terminates
+// abruptly.
+type SimpleFileFiler struct {
+	file *os.File
+	nest int
+	size int64 // not set if < 0
+}
+
+// NewSimpleFileFiler returns a new SimpleFileFiler.
+func NewSimpleFileFiler(f *os.File) *SimpleFileFiler {
+	return &SimpleFileFiler{file: f, size: -1}
+}
+
+// BeginUpdate implements Filer.
+func (f *SimpleFileFiler) BeginUpdate() error {
+	f.nest++
+	return nil
+}
+
+// Close implements Filer.
+func (f *SimpleFileFiler) Close() (err error) {
+	if f.nest != 0 {
+		return &ErrPERM{(f.Name() + ":Close")}
+	}
+
+	return f.file.Close()
+}
+
+// EndUpdate implements Filer.
+func (f *SimpleFileFiler) EndUpdate() (err error) {
+	if f.nest == 0 {
+		return &ErrPERM{(f.Name() + ":EndUpdate")}
+	}
+
+	f.nest--
+	return
+}
+
+// Name implements Filer.
+func (f *SimpleFileFiler) Name() string {
+	return f.file.Name()
+}
+
+// PunchHole implements Filer.
+func (f *SimpleFileFiler) PunchHole(off, size int64) (err error) {
+	return fileutil.PunchHole(f.file, off, size)
+}
+
+// ReadAt implements Filer.
+func (f *SimpleFileFiler) ReadAt(b []byte, off int64) (n int, err error) {
+	return f.file.ReadAt(b, off)
+}
+
+// Rollback implements Filer.
+func (f *SimpleFileFiler) Rollback() (err error) { return }
+
+// Size implements Filer.
+func (f *SimpleFileFiler) Size() (int64, error) {
+	if f.size < 0 { // boot
+		fi, err := os.Stat(f.file.Name())
+		if err != nil {
+			return 0, err
+		}
+
+		f.size = fi.Size()
+	}
+	return f.size, nil
+}
+
+// Sync implements Filer.
+func (f *SimpleFileFiler) Sync() error {
+	return f.file.Sync()
+}
+
+// Truncate implements Filer.
+func (f *SimpleFileFiler) Truncate(size int64) (err error) {
+	if size < 0 {
+		return &ErrINVAL{"Truncate size", size}
+	}
+
+	f.size = size
+	return f.file.Truncate(size)
+}
+
+// WriteAt implements Filer.
+func (f *SimpleFileFiler) WriteAt(b []byte, off int64) (n int, err error) {
+	if f.size < 0 { // boot
+		fi, err := os.Stat(f.file.Name())
+		if err != nil {
+			return 0, err
+		}
+
+		f.size = fi.Size()
+	}
+	f.size = mathutil.MaxInt64(f.size, int64(len(b))+off)
+	return f.file.WriteAt(b, off)
+}

Godeps/_workspace/src/github.com/cznic/exp/lldb/xact.go

@@ -0,0 +1,629 @@
+// Copyright 2014 The lldb Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Structural transactions.
+
+package lldb
+
+//DONE+ TransactionalMemoryFiler
+//	----
+//	Use NewRollbackFiler(myMemFiler, ...)
+
+/*
+
+bfBits: 3
+BenchmarkRollbackFiler	20000000	       102 ns/op	   9.73 MB/s
+
+bfBits: 4
+BenchmarkRollbackFiler	50000000	        55.7 ns/op	  17.95 MB/s
+
+bfBits: 5
+BenchmarkRollbackFiler	100000000	        32.2 ns/op	  31.06 MB/s
+
+bfBits: 6
+BenchmarkRollbackFiler	100000000	        20.6 ns/op	  48.46 MB/s
+
+bfBits: 7
+BenchmarkRollbackFiler	100000000	        15.1 ns/op	  66.12 MB/s
+
+bfBits: 8
+BenchmarkRollbackFiler	100000000	        10.5 ns/op	  95.66 MB/s
+
+bfBits: 9
+BenchmarkRollbackFiler	200000000	         8.02 ns/op	 124.74 MB/s
+
+bfBits: 10
+BenchmarkRollbackFiler	200000000	         9.25 ns/op	 108.09 MB/s
+
+bfBits: 11
+BenchmarkRollbackFiler	100000000	        11.7 ns/op	  85.47 MB/s
+
+bfBits: 12
+BenchmarkRollbackFiler	100000000	        17.2 ns/op	  57.99 MB/s
+
+bfBits: 13
+BenchmarkRollbackFiler	100000000	        32.7 ns/op	  30.58 MB/s
+
+bfBits: 14
+BenchmarkRollbackFiler	50000000	        39.6 ns/op	  25.27 MB/s
+
+*/
+
+import (
+	"fmt"
+	"io"
+	"sync"
+
+	"github.com/coreos/rocket/Godeps/_workspace/src/github.com/cznic/fileutil"
+	"github.com/coreos/rocket/Godeps/_workspace/src/github.com/cznic/mathutil"
+)
+
+var (
+	_ Filer = &bitFiler{}      // Ensure bitFiler is a Filer.
+	_ Filer = &RollbackFiler{} // ditto
+)
+
+const (
+	bfBits = 9
+	bfSize = 1 << bfBits
+	bfMask = bfSize - 1
+)
+
+var (
+	bitmask       = [8]byte{1, 2, 4, 8, 16, 32, 64, 128}
+	bitZeroPage   bitPage
+	allDirtyFlags [bfSize >> 3]byte
+)
+
+func init() {
+	for i := range allDirtyFlags {
+		allDirtyFlags[i] = 0xff
+	}
+}
+
+type (
+	bitPage struct {
+		prev, next *bitPage
+		data       [bfSize]byte
+		flags      [bfSize >> 3]byte
+		dirty      bool
+	}
+
+	bitFilerMap map[int64]*bitPage
+
+	bitFiler struct {
+		parent Filer
+		m      bitFilerMap
+		size   int64
+	}
+)
+
+func newBitFiler(parent Filer) (f *bitFiler, err error) {
+	sz, err := parent.Size()
+	if err != nil {
+		return
+	}
+
+	return &bitFiler{parent: parent, m: bitFilerMap{}, size: sz}, nil
+}
+
+func (f *bitFiler) BeginUpdate() error { panic("internal error") }
+func (f *bitFiler) EndUpdate() error   { panic("internal error") }
+func (f *bitFiler) Rollback() error    { panic("internal error") }
+func (f *bitFiler) Sync() error        { panic("internal error") }
+
+func (f *bitFiler) Close() (err error)   { return }
+func (f *bitFiler) Name() string         { return fmt.Sprintf("%p.bitfiler", f) }
+func (f *bitFiler) Size() (int64, error) { return f.size, nil }
+
+func (f *bitFiler) PunchHole(off, size int64) (err error) {
+	first := off >> bfBits
+	if off&bfMask != 0 {
+		first++
+	}
+	off += size - 1
+	last := off >> bfBits
+	if off&bfMask != 0 {
+		last--
+	}
+	if limit := f.size >> bfBits; last > limit {
+		last = limit
+	}
+	for pgI := first; pgI <= last; pgI++ {
+		pg := &bitPage{}
+		pg.flags = allDirtyFlags
+		f.m[pgI] = pg
+	}
+	return
+}
+
+func (f *bitFiler) ReadAt(b []byte, off int64) (n int, err error) {
+	avail := f.size - off
+	pgI := off >> bfBits
+	pgO := int(off & bfMask)
+	rem := len(b)
+	if int64(rem) >= avail {
+		rem = int(avail)
+		err = io.EOF
+	}
+	for rem != 0 && avail > 0 {
+		pg := f.m[pgI]
+		if pg == nil {
+			pg = &bitPage{}
+			if f.parent != nil {
+				_, err = f.parent.ReadAt(pg.data[:], off&^bfMask)
+				if err != nil && !fileutil.IsEOF(err) {
+					return
+				}
+
+				err = nil
+			}
+			f.m[pgI] = pg
+		}
+		nc := copy(b[:mathutil.Min(rem, bfSize)], pg.data[pgO:])
+		pgI++
+		pgO = 0
+		rem -= nc
+		n += nc
+		b = b[nc:]
+		off += int64(nc)
+	}
+	return
+}
+
+func (f *bitFiler) Truncate(size int64) (err error) {
+	switch {
+	case size < 0:
+		return &ErrINVAL{"Truncate size", size}
+	case size == 0:
+		f.m = bitFilerMap{}
+		f.size = 0
+		return
+	}
+
+	first := size >> bfBits
+	if size&bfMask != 0 {
+		first++
+	}
+	last := f.size >> bfBits
+	if f.size&bfMask != 0 {
+		last++
+	}
+	for ; first < last; first++ {
+		delete(f.m, first)
+	}
+
+	f.size = size
+	return
+}
+
+func (f *bitFiler) WriteAt(b []byte, off int64) (n int, err error) {
+	off0 := off
+	pgI := off >> bfBits
+	pgO := int(off & bfMask)
+	n = len(b)
+	rem := n
+	var nc int
+	for rem != 0 {
+		pg := f.m[pgI]
+		if pg == nil {
+			pg = &bitPage{}
+			if f.parent != nil {
+				_, err = f.parent.ReadAt(pg.data[:], off&^bfMask)
+				if err != nil && !fileutil.IsEOF(err) {
+					return
+				}
+
+				err = nil
+			}
+			f.m[pgI] = pg
+		}
+		nc = copy(pg.data[pgO:], b)
+		pgI++
+		pg.dirty = true
+		for i := pgO; i < pgO+nc; i++ {
+			pg.flags[i>>3] |= bitmask[i&7]
+		}
+		pgO = 0
+		rem -= nc
+		b = b[nc:]
+		off += int64(nc)
+	}
+	f.size = mathutil.MaxInt64(f.size, off0+int64(n))
+	return
+}
+
+func (f *bitFiler) link() {
+	for pgI, pg := range f.m {
+		nx, ok := f.m[pgI+1]
+		if !ok || !nx.dirty {
+			continue
+		}
+
+		nx.prev, pg.next = pg, nx
+	}
+}
+
+func (f *bitFiler) dumpDirty(w io.WriterAt) (nwr int, err error) {
+	f.link()
+	for pgI, pg := range f.m {
+		if !pg.dirty {
+			continue
+		}
+
+		for pg.prev != nil && pg.prev.dirty {
+			pg = pg.prev
+			pgI--
+		}
+
+		for pg != nil && pg.dirty {
+			last := false
+			var off int64
+			first := -1
+			for i := 0; i < bfSize; i++ {
+				flag := pg.flags[i>>3]&bitmask[i&7] != 0
+				switch {
+				case flag && !last: // Leading edge detected
+					off = pgI<<bfBits + int64(i)
+					first = i
+				case !flag && last: // Trailing edge detected
+					n, err := w.WriteAt(pg.data[first:i], off)
+					if n != i-first {
+						return 0, err
+					}
+					first = -1
+					nwr++
+				}
+
+				last = flag
+			}
+			if first >= 0 {
+				i := bfSize
+				n, err := w.WriteAt(pg.data[first:i], off)
+				if n != i-first {
+					return 0, err
+				}
+
+				nwr++
+			}
+
+			pg.dirty = false
+			pg = pg.next
+			pgI++
+		}
+	}
+	return
+}
+
+// RollbackFiler is a Filer implementing structural transaction handling.
+// Structural transactions should be small and short lived because all non
+// committed data are held in memory until committed or discarded by a
+// Rollback.
+//
+// While using RollbackFiler, every intended update of the wrapped Filler, by
+// WriteAt, Truncate or PunchHole, _must_ be made within a transaction.
+// Attempts to do it outside of a transaction will return ErrPERM. OTOH,
+// invoking ReadAt outside of a transaction is not a problem.
+//
+// No nested transactions: All updates within a transaction are held in memory.
+// On a matching EndUpdate the updates held in memory are actually written to
+// the wrapped Filer.
+//
+// Nested transactions: Correct data will be seen from RollbackFiler when any
+// level of a nested transaction is rollbacked. The actual writing to the
+// wrapped Filer happens only when the outer most transaction nesting level is
+// closed.
+//
+// Invoking Rollback is an alternative to EndUpdate. It discards all changes
+// made at the current transaction level and returns the "state" (possibly not
+// yet persisted) of the Filer to what it was before the corresponding
+// BeginUpdate.
+//
+// During an open transaction, all reads (using ReadAt) are "dirty" reads,
+// seeing the uncommitted changes made to the Filer's data.
+//
+// Lldb databases should be based upon a RollbackFiler.
+//
+// With a wrapped MemFiler one gets transactional memory. With, for example a
+// wrapped disk based SimpleFileFiler it protects against at least some HW
+// errors - if Rollback is properly invoked on such failures and/or if there's
+// some WAL or 2PC or whatever other safe mechanism based recovery procedure
+// used by the client.
+//
+// The "real" writes to the wrapped Filer (or WAL instead) go through the
+// writerAt supplied to NewRollbackFiler.
+//
+// List of functions/methods which are recommended to be wrapped in a
+// BeginUpdate/EndUpdate structural transaction:
+//
+// 	Allocator.Alloc
+// 	Allocator.Free
+// 	Allocator.Realloc
+//
+//	CreateBTree
+// 	RemoveBTree
+// 	BTree.Clear
+// 	BTree.Delete
+// 	BTree.DeleteAny
+// 	BTree.Clear
+// 	BTree.Extract
+// 	BTree.Get (it can mutate the DB)
+// 	BTree.Put
+// 	BTree.Set
+//
+// NOTE: RollbackFiler is a generic solution intended to wrap Filers provided
+// by this package which do not implement any of the transactional methods.
+// RollbackFiler thus _does not_ invoke any of the transactional methods of its
+// wrapped Filer.
+//
+// RollbackFiler is safe for concurrent use by multiple goroutines.
+type RollbackFiler struct {
+	mu           sync.RWMutex
+	inCallback   bool
+	inCallbackMu sync.RWMutex
+	bitFiler     *bitFiler
+	checkpoint   func(int64) error
+	closed       bool
+	f            Filer
+	parent       Filer
+	tlevel       int // transaction nesting level, 0 == not in transaction
+	writerAt     io.WriterAt
+
+	// afterRollback, if not nil, is called after performing Rollback
+	// without errros.
+	afterRollback func() error
+}
+
+// NewRollbackFiler returns a RollbackFiler wrapping f.
+//
+// The checkpoint parameter
+//
+// The checkpoint function is called after closing (by EndUpdate) the upper
+// most level open transaction if all calls of writerAt were successful and the
+// DB (or eg. a WAL) is thus now in a consistent state (virtually, in the ideal
+// world with no write caches, no HW failures, no process crashes, ...).
+//
+// NOTE: In, for example, a 2PC it is necessary to reflect also the sz
+// parameter as the new file size (as in the parameter to Truncate). All
+// changes were successfully written already by writerAt before invoking
+// checkpoint.
+//
+// The writerAt parameter
+//
+// The writerAt interface is used to commit the updates of the wrapped Filer.
+// If any invocation of writerAt fails then a non nil error will be returned
+// from EndUpdate and checkpoint will _not_ ne called.  Neither is necessary to
+// call Rollback. The rule of thumb: The [structural] transaction [level] is
+// closed by invoking exactly once one of EndUpdate _or_ Rollback.
+//
+// It is presumed that writerAt uses WAL or 2PC or whatever other safe
+// mechanism to physically commit the updates.
+//
+// Updates performed by invocations of writerAt are byte-precise, but not
+// necessarily maximum possible length precise. IOW, for example an update
+// crossing page boundaries may be performed by more than one writerAt
+// invocation.  No offset sorting is performed.  This may change if it proves
+// to be a problem. Such change would be considered backward compatible.
+//
+// NOTE: Using RollbackFiler, but failing to ever invoke a matching "closing"
+// EndUpdate after an "opening" BeginUpdate means neither writerAt or
+// checkpoint will ever get called - with all the possible data loss
+// consequences.
+func NewRollbackFiler(f Filer, checkpoint func(sz int64) error, writerAt io.WriterAt) (r *RollbackFiler, err error) {
+	if f == nil || checkpoint == nil || writerAt == nil {
+		return nil, &ErrINVAL{Src: "lldb.NewRollbackFiler, nil argument"}
+	}
+
+	return &RollbackFiler{
+		checkpoint: checkpoint,
+		f:          f,
+		writerAt:   writerAt,
+	}, nil
+}
+
+// Implements Filer.
+func (r *RollbackFiler) BeginUpdate() (err error) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	parent := r.f
+	if r.tlevel != 0 {
+		parent = r.bitFiler
+	}
+	r.bitFiler, err = newBitFiler(parent)
+	if err != nil {
+		return
+	}
+
+	r.tlevel++
+	return
+}
+
+// Implements Filer.
+//
+// Close will return an error if not invoked at nesting level 0.  However, to
+// allow emergency closing from eg. a signal handler; if Close is invoked
+// within an open transaction(s), it rollbacks any non committed open
+// transactions and performs the Close operation.
+//
+// IOW: Regardless of the transaction nesting level the Close is always
+// performed but any uncommitted transaction data are lost.
+func (r *RollbackFiler) Close() (err error) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	if r.closed {
+		return &ErrPERM{r.f.Name() + ": Already closed"}
+	}
+
+	r.closed = true
+	if err = r.f.Close(); err != nil {
+		return
+	}
+
+	if r.tlevel != 0 {
+		err = &ErrPERM{r.f.Name() + ": Close inside an open transaction"}
+	}
+
+	return
+}
+
+// Implements Filer.
+func (r *RollbackFiler) EndUpdate() (err error) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	if r.tlevel == 0 {
+		return &ErrPERM{r.f.Name() + " : EndUpdate outside of a transaction"}
+	}
+
+	sz, err := r.size() // Cannot call .Size() -> deadlock
+	if err != nil {
+		return
+	}
+
+	r.tlevel--
+	bf := r.bitFiler
+	parent := bf.parent
+	w := r.writerAt
+	if r.tlevel != 0 {
+		w = parent
+	}
+	nwr, err := bf.dumpDirty(w)
+	if err != nil {
+		return
+	}
+
+	switch {
+	case r.tlevel == 0:
+		r.bitFiler = nil
+		if nwr == 0 {
+			return
+		}
+
+		return r.checkpoint(sz)
+	default:
+		r.bitFiler = parent.(*bitFiler)
+		sz, _ := bf.Size() // bitFiler.Size() never returns err != nil
+		return parent.Truncate(sz)
+	}
+}
+
+// Implements Filer.
+func (r *RollbackFiler) Name() string {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+
+	return r.f.Name()
+}
+
+// Implements Filer.
+func (r *RollbackFiler) PunchHole(off, size int64) error {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	if r.tlevel == 0 {
+		return &ErrPERM{r.f.Name() + ": PunchHole outside of a transaction"}
+	}
+
+	if off < 0 {
+		return &ErrINVAL{r.f.Name() + ": PunchHole off", off}
+	}
+
+	if size < 0 || off+size > r.bitFiler.size {
+		return &ErrINVAL{r.f.Name() + ": PunchHole size", size}
+	}
+
+	return r.bitFiler.PunchHole(off, size)
+}
+
+// Implements Filer.
+func (r *RollbackFiler) ReadAt(b []byte, off int64) (n int, err error) {
+	r.inCallbackMu.RLock()
+	defer r.inCallbackMu.RUnlock()
+	if !r.inCallback {
+		r.mu.RLock()
+		defer r.mu.RUnlock()
+	}
+	if r.tlevel == 0 {
+		return r.f.ReadAt(b, off)
+	}
+
+	return r.bitFiler.ReadAt(b, off)
+}
+
+// Implements Filer.
+func (r *RollbackFiler) Rollback() (err error) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	if r.tlevel == 0 {
+		return &ErrPERM{r.f.Name() + ": Rollback outside of a transaction"}
+	}
+
+	if r.tlevel > 1 {
+		r.bitFiler = r.bitFiler.parent.(*bitFiler)
+	}
+	r.tlevel--
+	if f := r.afterRollback; f != nil {
+		r.inCallbackMu.Lock()
+		r.inCallback = true
+		r.inCallbackMu.Unlock()
+		defer func() {
+			r.inCallbackMu.Lock()
+			r.inCallback = false
+			r.inCallbackMu.Unlock()
+		}()
+		return f()
+	}
+	return
+}
+
+func (r *RollbackFiler) size() (sz int64, err error) {
+	if r.tlevel == 0 {
+		return r.f.Size()
+	}
+
+	return r.bitFiler.Size()
+}
+
+// Implements Filer.
+func (r *RollbackFiler) Size() (sz int64, err error) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	return r.size()
+}
+
+// Implements Filer.
+func (r *RollbackFiler) Sync() error {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	return r.f.Sync()
+}
+
+// Implements Filer.
+func (r *RollbackFiler) Truncate(size int64) error {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	if r.tlevel == 0 {
+		return &ErrPERM{r.f.Name() + ": Truncate outside of a transaction"}
+	}
+
+	return r.bitFiler.Truncate(size)
+}
+
+// Implements Filer.
+func (r *RollbackFiler) WriteAt(b []byte, off int64) (n int, err error) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	if r.tlevel == 0 {
+		return 0, &ErrPERM{r.f.Name() + ": WriteAt outside of a transaction"}
+	}
+
+	return r.bitFiler.WriteAt(b, off)
+}

Godeps/_workspace/src/github.com/cznic/exp/lldb/xact_test.go

@@ -0,0 +1,400 @@
+// Copyright 2014 The lldb Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package lldb
+
+import (
+	"bytes"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"math/rand"
+	"testing"
+
+	"github.com/coreos/rocket/Godeps/_workspace/src/github.com/cznic/fileutil"
+	"github.com/coreos/rocket/Godeps/_workspace/src/github.com/cznic/mathutil"
+)
+
+func (f *bitFiler) dump(w io.Writer) {
+	fmt.Fprintf(w, "bitFiler @ %p, size: %d(%#x)\n", f, f.size, f.size)
+	for k, v := range f.m {
+		fmt.Fprintf(w, "bitPage @ %p: pgI %d(%#x): %#v\n", v, k, k, *v)
+	}
+}
+
+func filerBytes(f Filer) []byte {
+	sz, err := f.Size()
+	if err != nil {
+		panic(err)
+	}
+
+	b := make([]byte, int(sz))
+	n, err := f.ReadAt(b, 0)
+	if n != len(b) {
+		panic(fmt.Errorf("sz %d n %d err %v", sz, n, err))
+	}
+
+	return b
+}
+
+func cmpFilerBytes(t *testing.T, fg, fe Filer) {
+	g, e := filerBytes(fg), filerBytes(fe)
+	if !bytes.Equal(g, e) {
+		t.Fatalf("Filer content doesn't match: got\n%sexp:\n%s", hex.Dump(g), hex.Dump(e))
+	}
+}
+
+func TestRollbackFiler0(t *testing.T) {
+	var r *RollbackFiler
+	f, g := NewMemFiler(), NewMemFiler()
+
+	checkpoint := func(sz int64) (err error) {
+		return f.Truncate(sz)
+	}
+
+	r, err := NewRollbackFiler(f, checkpoint, f)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err = r.BeginUpdate(); err != nil {
+		t.Fatal(err)
+	}
+
+	if err = r.EndUpdate(); err != nil {
+		t.Fatal(err)
+	}
+
+	cmpFilerBytes(t, f, g)
+}
+
+func TestRollbackFiler1(t *testing.T) {
+	const (
+		N = 1e6
+		O = 1234
+	)
+
+	var r *RollbackFiler
+	f, g := NewMemFiler(), NewMemFiler()
+
+	checkpoint := func(sz int64) (err error) {
+		return f.Truncate(sz)
+	}
+
+	r, err := NewRollbackFiler(f, checkpoint, f)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err = r.BeginUpdate(); err != nil {
+		t.Fatal(err)
+	}
+
+	rng := rand.New(rand.NewSource(42))
+	b := make([]byte, N)
+	for i := range b {
+		b[i] = byte(rng.Int())
+	}
+
+	if _, err = g.WriteAt(b, O); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err = r.WriteAt(b, O); err != nil {
+		t.Fatal(err)
+	}
+
+	b = filerBytes(f)
+	if n := len(b); n != 0 {
+		t.Fatal(n)
+	}
+
+	if err = r.EndUpdate(); err != nil {
+		t.Fatal(err)
+	}
+
+	cmpFilerBytes(t, f, g)
+}
+
+func TestRollbackFiler2(t *testing.T) {
+	const (
+		N = 1e6
+		O = 1234
+	)
+
+	var r *RollbackFiler
+	f, g := NewMemFiler(), NewMemFiler()
+
+	checkpoint := func(sz int64) (err error) {
+		return f.Truncate(sz)
+	}
+
+	r, err := NewRollbackFiler(f, checkpoint, f)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err = r.BeginUpdate(); err != nil {
+		t.Fatal(err)
+	}
+
+	rng := rand.New(rand.NewSource(42))
+	b := make([]byte, N)
+	for i := range b {
+		b[i] = byte(rng.Int())
+	}
+
+	if _, err = r.WriteAt(b, O); err != nil {
+		t.Fatal(err)
+	}
+
+	b = filerBytes(f)
+	if n := len(b); n != 0 {
+		t.Fatal(n)
+	}
+
+	if err = r.Rollback(); err != nil {
+		t.Fatal(err)
+	}
+
+	cmpFilerBytes(t, f, g)
+}
+
+func rndBytes(rng *rand.Rand, n int) []byte {
+	r := make([]byte, n)
+	for i := range r {
+		r[i] = byte(rng.Int())
+	}
+	return r
+}
+
+func TestRollbackFiler3(t *testing.T) {
+	var r *RollbackFiler
+	f := NewMemFiler()
+
+	checkpoint := func(sz int64) (err error) {
+		return f.Truncate(sz)
+	}
+
+	r, err := NewRollbackFiler(f, checkpoint, f)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	n, err := r.ReadAt([]byte{0}, 0)
+	if n != 0 || !fileutil.IsEOF(err) {
+		t.Fatal(n, err)
+	}
+
+	n, err = r.ReadAt([]byte{0}, 1e6)
+	if n != 0 || !fileutil.IsEOF(err) {
+		t.Fatal(n, err)
+	}
+
+	if err = r.BeginUpdate(); err != nil { // BeginUpdate: 0 -> 1
+		t.Fatal(err)
+	}
+
+	rng := rand.New(rand.NewSource(42))
+
+	buf := rndBytes(rng, 100)
+	if n, err := r.WriteAt(buf, 1e6); n != 100 || err != nil {
+		t.Fatal(err)
+	}
+
+	buf = make([]byte, 100)
+	if n, err := r.ReadAt(buf, 1e6-200); n != 100 || err != nil {
+		t.Fatal(err)
+	}
+
+	for i, v := range buf {
+		if v != 0 {
+			t.Fatal(i, v)
+		}
+	}
+
+	if err := r.Truncate(1e5); err != nil {
+		t.Fatal(err)
+	}
+
+	if err = r.BeginUpdate(); err != nil { // BeginUpdate: 1 -> 2
+		t.Fatal(err)
+	}
+
+	if n, err := r.ReadAt(buf, 1e6); n != 0 || err == nil {
+		t.Fatal(n, err)
+	}
+
+	if err := r.Truncate(2e6); err != nil {
+		t.Fatal(err)
+	}
+
+	if err = r.BeginUpdate(); err != nil { // BeginUpdate: 2 -> 3
+		t.Fatal(err)
+	}
+
+	if n, err := r.ReadAt(buf, 1e6); n == 0 || err != nil {
+		t.Fatal(n, err)
+	}
+
+	for i, v := range buf {
+		if v != 0 {
+			t.Fatal(i, v)
+		}
+	}
+}
+
+func TestRollbackFiler4(t *testing.T) {
+	const (
+		maxSize    = 1e6
+		maxChange  = maxSize/100 + 4
+		maxChanges = 10
+		maxNest    = 3
+	)
+
+	var r *RollbackFiler
+	f := NewMemFiler()
+
+	checkpoint := func(sz int64) (err error) {
+		return f.Truncate(sz)
+	}
+
+	r, err := NewRollbackFiler(f, checkpoint, f)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	rng := rand.New(rand.NewSource(42))
+
+	ref := make([]byte, 2*maxSize)
+	for i := range ref {
+		ref[i] = byte(rng.Int())
+	}
+
+	var finalSize int
+
+	var fn func(int, int, []byte) (int, []byte)
+	fn = func(nest, inSize int, in []byte) (outSize int, out []byte) {
+		defer func() {
+			for i := outSize; i < len(out); i++ {
+				out[i] = 0
+			}
+			finalSize = mathutil.Max(finalSize, outSize)
+		}()
+
+		out = make([]byte, len(in), 2*maxSize)
+		copy(out, in)
+		if err := r.BeginUpdate(); err != nil {
+			t.Fatal(err)
+		}
+
+		for i := 0; i < maxChanges; i++ {
+			changeLen := rng.Intn(maxChange) + 4
+			changeOff := rng.Intn(maxSize * 3 / 2)
+			b := make([]byte, changeLen)
+			for i := range b {
+				b[i] = byte(rng.Int())
+			}
+			if n, err := r.WriteAt(b, int64(changeOff)); n != len(b) || err != nil {
+				t.Fatal(n, len(b), err)
+			}
+		}
+
+		if err := r.Rollback(); err != nil {
+			t.Fatal(err)
+		}
+
+		if err := r.BeginUpdate(); err != nil {
+			t.Fatal(err)
+		}
+
+		for i := 0; i < maxChanges; i++ {
+			changeLen := rng.Intn(maxChange) + 4
+			changeOff := rng.Intn(maxSize * 3 / 2)
+			b := make([]byte, changeLen)
+			for i := range b {
+				b[i] = byte(rng.Int())
+			}
+			if n, err := r.WriteAt(b, int64(changeOff)); n != len(b) || err != nil {
+				t.Fatal(n, len(b), err)
+			}
+			copy(out[changeOff:], b)
+			copy(ref[changeOff:], b)
+		}
+
+		newSize := rng.Intn(maxSize*3/2) + 4
+		if nest == maxNest {
+			if err := r.EndUpdate(); err != nil {
+				t.Fatal(err)
+			}
+
+			return newSize, out
+		}
+
+		outSize, out = fn(nest+1, newSize, out)
+		if err := r.EndUpdate(); err != nil {
+			t.Fatal(err)
+		}
+
+		return
+	}
+
+	sz, result := fn(0, maxSize, ref)
+	if g, e := sz, finalSize; g != e {
+		t.Fatal(err)
+	}
+
+	g, e := result[:sz], ref[:sz]
+	if !bytes.Equal(g, e) {
+		if len(g) == len(e) {
+			x := make([]byte, len(g))
+			for i := range x {
+				if g[i] != e[i] {
+					x[i] = 'X'
+				}
+			}
+			//t.Logf("Data diff\n%s", hex.Dump(x))
+		}
+		//t.Fatalf("Data don't match: got\n%sexp:\n%s", hex.Dump(g), hex.Dump(e))
+		t.Fatalf("Data don't match")
+	}
+}
+
+func BenchmarkRollbackFiler(b *testing.B) {
+	rng := rand.New(rand.NewSource(42))
+	type t struct {
+		off int64
+		b   []byte
+	}
+	a := []t{}
+	for rem := b.N; rem > 0; {
+		off := rng.Int63()
+		n := mathutil.Min(rng.Intn(1e3)+1, rem)
+		a = append(a, t{off, rndBytes(rng, n)})
+		rem -= n
+	}
+
+	var r *RollbackFiler
+	f := NewMemFiler()
+
+	checkpoint := func(sz int64) (err error) {
+		return f.Truncate(sz)
+	}
+
+	r, err := NewRollbackFiler(f, checkpoint, f)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	if err := r.BeginUpdate(); err != nil {
+		b.Fatal(err)
+	}
+
+	b.ResetTimer()
+	for _, v := range a {
+		if _, err := r.WriteAt(v.b, v.off); err != nil {
+			b.Fatal(err)
+		}
+	}
+}

Godeps/_workspace/src/github.com/cznic/fileutil/AUTHORS

@@ -0,0 +1,14 @@
+# This file lists authors for copyright purposes.  This file is distinct from
+# the CONTRIBUTORS files.  See the latter for an explanation.
+#
+# Names should be added to this file as:
+#     Name or Organization <email address>
+#
+# The email address is not required for organizations.
+#
+# Please keep the list sorted.
+
+CZ.NIC z.s.p.o. <kontakt@nic.cz>
+Jan Mercl <0xjnml@gmail.com>
+Aaron Bieber <deftly@gmail.com>
+

Godeps/_workspace/src/github.com/cznic/fileutil/CONTRIBUTORS

@@ -0,0 +1,14 @@
+# This file lists people who contributed code to this repository.  The AUTHORS
+# file lists the copyright holders; this file lists people.
+#
+# Names should be added to this file like so:
+#     Name <email address>
+#
+# Please keep the list sorted.
+
+Bill Thiede <xinu.tv>
+Gary Burd <gary@beagledreams.com>
+Jan Mercl <0xjnml@gmail.com>
+Nick Owens <mischief@offblast.org>
+Tamás Gulácsi <gt-dev@gthomas.eu>
+Aaron Bieber <deftly@gmail.com>

Godeps/_workspace/src/github.com/cznic/fileutil/LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2014 The fileutil Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the names of the authors nor the names of the
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

Godeps/_workspace/src/github.com/cznic/fileutil/Makefile

@@ -0,0 +1,27 @@
+# Copyright (c) 2014 The fileutil authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+.PHONY: all clean editor todo
+
+all: editor
+	go vet
+	golint .
+	go install
+	make todo
+
+editor:
+	go fmt
+	go test -i
+	go test
+	go build
+
+todo:
+	@grep -n ^[[:space:]]*_[[:space:]]*=[[:space:]][[:alpha:]][[:alnum:]]* *.go || true
+	@grep -n TODO *.go || true
+	@grep -n BUG *.go || true
+	@grep -n println *.go || true
+
+clean:
+	@go clean
+	rm -f y.output

Godeps/_workspace/src/github.com/cznic/fileutil/README

@@ -0,0 +1,16 @@
+This is a goinstall-able mirror of modified code already published at:
+http://git.nic.cz/redmine/projects/gofileutil/repository
+
+Packages in this repository:
+
+Install: $go get github.com/cznic/fileutil
+Godocs: http://godoc.org/github.com/cznic/fileutil
+
+Install: $go get github.com/cznic/fileutil/storage
+Godocs: http://godoc.org/github.com/cznic/fileutil/storage
+
+Install: $go get github.com/cznic/fileutil/falloc
+Godocs: http://godoc.org/github.com/cznic/fileutil/falloc
+
+Install: $go get github.com/cznic/fileutil/hdb
+Godocs: http://godoc.org/github.com/cznic/fileutil/hdb

Godeps/_workspace/src/github.com/cznic/fileutil/all_test.go

@@ -0,0 +1,39 @@
+// Copyright (c) 2014 The fileutil Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package fileutil
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func TestTempFile(t *testing.T) {
+	f, err := TempFile("", "abc", "mno.xyz")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	n := f.Name()
+	t.Log(n)
+	defer func() {
+		f.Close()
+		os.Remove(n)
+	}()
+
+	base := filepath.Base(n)
+	if base == "abcmno.xyz" {
+		t.Fatal(base)
+	}
+
+	if !strings.HasPrefix(base, "abc") {
+		t.Fatal(base)
+	}
+
+	if !strings.HasSuffix(base, "mno.xyz") {
+		t.Fatal(base)
+	}
+}

Godeps/_workspace/src/github.com/cznic/fileutil/falloc/LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of CZ.NIC nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

Godeps/_workspace/src/github.com/cznic/fileutil/falloc/README

@@ -0,0 +1,5 @@
+This is a goinstall-able mirror of modified code already published at:
+https://git.nic.cz/redmine/projects/gofileutil/repository/show/falloc
+
+Install: $go get github.com/cznic/fileutil/falloc
+Godocs: http://gopkgdoc.appspot.com/pkg/github.com/cznic/fileutil/falloc

Godeps/_workspace/src/github.com/cznic/fileutil/falloc/docs.go

@@ -0,0 +1,251 @@
+// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// blame: jnml, labs.nic.cz
+
+/*
+
+WIP: Package falloc provides allocation/deallocation of space within a
+file/store (WIP, unstable API).
+
+Overall structure:
+ File == n blocks.
+ Block == n atoms.
+ Atom == 16 bytes.
+
+x6..x0 == least significant 7 bytes of a 64 bit integer, highest (7th) byte is
+0 and is not stored in the file.
+
+Block first byte
+
+Aka block type tag.
+
+------------------------------------------------------------------------------
+
+0xFF: Free atom (free block of size 1).
+ +------++---------++---------++------+
+ |  0   ||  1...7  || 8...14  ||  15  |
+ +------++---------++---------++------+
+ | 0xFF || p6...p0 || n6...n0 || 0xFF |
+ +------++---------++---------++------+
+
+Link to the previous free block (atom addressed) is p6...p0, next dtto in
+n6...n0.  Doubly linked lists of "compatible" free blocks allows for free space
+reclaiming and merging.  "Compatible" == of size at least some K. Heads of all
+such lists are organized per K or intervals of Ks elsewhere.
+
+------------------------------------------------------------------------------
+
+0xFE: Free block, size == s6...s0 atoms.
+ +------++---------++---------++---------++--
+ |  +0  ||  1...7  || 8...14  || 15...21 || 22...16*size-1
+ +------++---------++---------++---------++--
+ | 0xFE || p6...p0 || n6...n0 || s6...s0 || ...
+ +------++---------++---------++---------++--
+
+Prev and next links as in the 0xFF first byte case.  End of this block - see
+"Block last byte": 0xFE bellow. Data between == undefined.
+
+------------------------------------------------------------------------------
+
+0xFD:  Relocated block.
+ +------++---------++-----------++------+
+ |  0   ||  1...7  ||  8...14   ||  15  |
+ +------++---------++-----------++------+
+ | 0xFD || r6...r0 || undefined || 0x00 | // == used block
+ +------++---------++-----------++------+
+
+Relocation link is r6..r0 == atom address. Relocations MUST NOT chain and MUST
+point to a "content" block, i.e. one with the first byte in 0x00...0xFC.
+
+Relocated block allows to permanently assign a handle/file pointer ("atom"
+address) to some content and resize the content anytime afterwards w/o having
+to update all the possible existing references to the original handle.
+
+------------------------------------------------------------------------------
+
+0xFC: Used long block.
+ +------++---------++--------------------++---------+---+
+ |  0   ||  1...2  ||      3...N+2       ||         |   |
+ +------++---------++--------------------++---------+---+
+ | 0xFC || n1...n0 || N bytes of content || padding | Z |
+ +------++---------++--------------------++---------+---+
+
+This block type is used for content of length in N == 238...61680 bytes. N is
+encoded as a 2 byte unsigned integer n1..n0 in network byte order. Values
+bellow 238 are reserved, those content lengths are to be carried by the
+0x00..0xFB block types.
+
+ 1. n in 0x00EE...0xF0F0 is used for content under the same rules
+    as in the 0x01..0xED type.
+
+ 2. If the last byte of the content is not the last byte of an atom then
+    the last byte of the block is 0x00.
+
+ 3. If the last byte of the content IS the last byte of an atom:
+
+   3.1 If the last byte of content is in 0x00..0xFD then everything is OK.
+
+   3.2 If the last byte of content is 0xFE or 0xFF then the escape
+       via n > 0xF0F0 MUST be used AND the block's last byte is 0x00 or 0x01,
+       meaning value 0xFE and 0xFF respectively.
+
+ 4. n in 0xF0F1...0xFFFF is like the escaped 0xEE..0xFB block.
+    N == 13 + 16(n - 0xF0F1).
+
+Discussion of the padding and Z fields - see the 0x01..0xED block type.
+
+------------------------------------------------------------------------------
+
+0xEE...0xFB: Used escaped short block.
+ +---++----------------------++---+
+ | 0 ||        1...N-1       ||   |
+ +---++----------------------++---+
+ | X || N-1 bytes of content || Z |
+ +---++----------------------++---+
+
+N == 15 + 16(X - 0xEE). Z is the content last byte encoded as follows.
+
+case Z == 0x00:	The last byte of content is 0xFE
+
+case Z == 0x01:	The last byte of content is 0xFF
+
+------------------------------------------------------------------------------
+
+0x01...0xED: Used short block.
+ +---++--------------------++---------+---+
+ | 0 ||        1...N       ||         |   |
+ +---++--------------------++---------+---+
+ | N || N bytes of content || padding | Z |
+ +---++--------------------++---------+---+
+
+This block type is used for content of length in 1...237 bytes.  The value of
+the "padding" field, if of non zero length, is undefined.
+
+If the last byte of content is the last byte of an atom (== its file byte
+offset & 0xF == 0xF) then such last byte MUST be in 0x00...0xFD.
+
+If the last byte of content is the last byte of an atom AND the last byte of
+content is 0xFE or 0xFF then the short escape block type (0xEE...0xFB) MUST be
+used.
+
+If the last byte of content is not the last byte of an atom, then the last byte
+of such block, i.e. the Z field, which is also a last byte of some atom, MUST
+be 0x00 (i.e. the used block marker).  Other "tail" values are reserved.
+
+------------------------------------------------------------------------------
+
+0x00: Used empty block.
+ +------++-----------++------+
+ |  0   ||  1...14   ||  15  |
+ +------++-----------++------+
+ | 0x00 || undefined || 0x00 | // == used block, other "tail" values reserved.
+ +------++-----------++------+
+
+All of the rules for 0x01..0xED applies. Depicted only for its different
+semantics (e.g. an allocated [existing] string but with length of zero).
+
+==============================================================================
+
+Block last byte
+
+------------------------------------------------------------------------------
+
+0xFF: Free atom. Layout - see "Block first byte": FF.
+
+------------------------------------------------------------------------------
+
+0xFE: Free block, size n atoms. Preceding 7 bytes == size (s6...s0) of the free
+block in atoms, network byte order
+   --++---------++------+
+     || -8...-2 ||  -1  |
+   --++---------++------+
+ ... || s6...s0 || 0xFE | <- block's last byte
+   --++---------++------+
+
+Layout at start of this block - see "Block first byte": FE.
+
+------------------------------------------------------------------------------
+
+0x00...0xFD: Used (non free) block.
+
+==============================================================================
+
+Free lists table
+
+The free lists table content is stored in the standard layout of a used block.
+
+A table item is a 7 byte size field followed by a 7 byte atom address field
+(both in network byte order), thus every item is 14 contiguous bytes. The
+item's address field is pointing to a free block.  The size field determines
+the minimal size (in atoms) of free blocks on that list.
+
+The free list table is n above items, thus the content has 14n bytes. Note that
+the largest block content is 61680 bytes and as there are 14 bytes per table
+item, so the table is limited to at most 4405 entries.
+
+Items in the table do not have to be sorted according to their size field values.
+
+No two items can have the same value of the size field.
+
+When freeing blocks, the block MUST be linked into an item list with the
+highest possible size field, which is less or equal to the number of atoms in
+the new free block.
+
+When freeing a block, the block MUST be first merged with any adjacent free
+blocks (thus possibly creating a bigger free block) using information derived
+from the adjacent blocks first and last bytes. Such merged free blocks MUST be
+removed from their original doubly linked lists. Afterwards the new bigger free
+block is put to the free list table in the appropriate item.
+
+Items with address field == 0 are legal. Such item is a placeholder for a empty
+list of free blocks of the item's size.
+
+Items with size field == 0 are legal. Such item is a placeholder, used e.g. to
+avoid further reallocations/redirecting of the free lists table.
+
+The largest possible allocation request (for content length 61680 bytes) is
+0xF10 (3856) atoms.  All free blocks of this or bigger size are presumably put
+into a single table item with the size 3856. It may be useful to additionally
+have a free lists table item which links free blocks of some bigger size (say
+1M+) and then use the OS sparse file support (if present) to save the physical
+space used by such free blocks.
+
+Smaller (<3856 atoms) free blocks can be organized exactly (every distinct size
+has its table item) or the sizes can run using other schema like e.g. "1, 2,
+4, 8, ..." (powers of 2) or "1, 2, 3, 5, 8, 13, ..." (the Fibonacci sequence)
+or they may be fine tuned to a specific usage pattern.
+
+==============================================================================
+
+Header
+
+The first block of a file (atom address == file offset == 0) is the file header.
+The header block has the standard layout of a used short non escaped block.
+
+Special conditions apply: The header block and its content MUST be like this:
+
+ +------+---------+---------+------+
+ |  0   |  1...7  | 8...14  |  15  |
+ +------+---------+---------+------+
+ | 0x0F | m6...m0 | f6...f0 | FLTT |
+ +------+---------+---------+------+
+
+m6..m0 is a "magic" value 0xF1C1A1FE51B1E.
+
+f6...f0 is the atom address of the free lists table (discussed elsewhere).
+If f6...f0 == 0x00 the there is no free lists table (yet).
+
+FLTT describes the type of the Free List Table. Currently defined values:
+
+------------------------------------------------------------------------------
+
+FLTT == 0: Free List Table is fixed at atom address 2. It has a fixed size for 3856 entries
+for free list of size 1..3855 atoms and the last is for the list of free block >= 3856 atoms.
+*/
+package falloc
+
+const (
+	INVALID_HANDLE = Handle(-1)
+)

Godeps/_workspace/src/github.com/cznic/fileutil/falloc/error.go

@@ -0,0 +1,130 @@
+// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// blame: jnml, labs.nic.cz
+
+package falloc
+
+import "fmt"
+
+// EBadRequest is an error produced for invalid operation, e.g. for data of more than maximum allowed.
+type EBadRequest struct {
+	Name string
+	Size int
+}
+
+func (e *EBadRequest) Error() string {
+	return fmt.Sprintf("%s: size %d", e.Name, e.Size)
+}
+
+// EClose is a file/store close error.
+type EClose struct {
+	Name string
+	Err  error
+}
+
+func (e *EClose) Error() string {
+	return fmt.Sprintf("%sx: %s", e.Name, e.Err)
+}
+
+// ECorrupted is a file/store format error.
+type ECorrupted struct {
+	Name string
+	Ofs  int64
+}
+
+func (e *ECorrupted) Error() string {
+	return fmt.Sprintf("%s: corrupted data @%#x", e.Name, e.Ofs)
+}
+
+// ECreate is a file/store create error.
+type ECreate struct {
+	Name string
+	Err  error
+}
+
+func (e *ECreate) Error() string {
+	return fmt.Sprintf("%s: %s", e.Name, e.Err)
+}
+
+// EFreeList is a file/store format error.
+type EFreeList struct {
+	Name  string
+	Size  int64
+	Block int64
+}
+
+func (e *EFreeList) Error() string {
+	return fmt.Sprintf("%s: invalid free list item, size %#x, block %#x", e.Name, e.Size, e.Block)
+}
+
+// EHandle is an error type reported for invalid Handles.
+type EHandle struct {
+	Name   string
+	Handle Handle
+}
+
+func (e EHandle) Error() string {
+	return fmt.Sprintf("%s: invalid handle %#x", e.Name, e.Handle)
+}
+
+// EHeader is a file/store format error.
+type EHeader struct {
+	Name     string
+	Header   []byte
+	Expected []byte
+}
+
+func (e *EHeader) Error() string {
+	return fmt.Sprintf("%s: invalid header, got [% x], expected [% x]", e.Name, e.Header, e.Expected)
+}
+
+// ENullHandle is a file/store access error via a null handle.
+type ENullHandle string
+
+func (e ENullHandle) Error() string {
+	return fmt.Sprintf("%s: access via null handle", e)
+}
+
+// EOpen is a file/store open error.
+type EOpen struct {
+	Name string
+	Err  error
+}
+
+func (e *EOpen) Error() string {
+	return fmt.Sprintf("%s: %s", e.Name, e.Err)
+}
+
+// ERead is a file/store read error.
+type ERead struct {
+	Name string
+	Ofs  int64
+	Err  error
+}
+
+func (e *ERead) Error() string {
+	return fmt.Sprintf("%s, %#x: %s", e.Name, e.Ofs, e.Err)
+}
+
+// ESize is a file/store size error.
+type ESize struct {
+	Name string
+	Size int64
+}
+
+func (e *ESize) Error() string {
+	return fmt.Sprintf("%s: invalid size %#x(%d), size %%16 != 0", e.Name, e.Size, e.Size)
+}
+
+// EWrite is a file/store write error.
+type EWrite struct {
+	Name string
+	Ofs  int64
+	Err  error
+}
+
+func (e *EWrite) Error() string {
+	return fmt.Sprintf("%s, %#x: %s", e.Name, e.Ofs, e.Err)
+}

Godeps/_workspace/src/github.com/cznic/fileutil/falloc/falloc.go

@@ -0,0 +1,676 @@
+// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// blame: jnml, labs.nic.cz
+
+/*
+
+This is an mostly (WIP) conforming implementation of the "specs" in docs.go.
+
+The main incompletness is support for only one kind of FTL, though this table kind is still per "specs".
+
+*/
+
+package falloc
+
+import (
+	"bytes"
+	"github.com/coreos/rocket/Godeps/_workspace/src/github.com/cznic/fileutil/storage"
+	"sync"
+)
+
+// Handle is a reference to a block in a file/store.
+// Handle is an uint56 wrapped in an in64, i.e. the most significant byte must be always zero.
+type Handle int64
+
+// Put puts the 7 least significant bytes of h into b. The MSB of h should be zero.
+func (h Handle) Put(b []byte) {
+	for ofs := 6; ofs >= 0; ofs-- {
+		b[ofs] = byte(h)
+		h >>= 8
+	}
+}
+
+// Get gets the 7 least significant bytes of h from b. The MSB of h is zeroed.
+func (h *Handle) Get(b []byte) {
+	var x Handle
+	for ofs := 0; ofs <= 6; ofs++ {
+		x = x<<8 | Handle(b[ofs])
+	}
+	*h = x
+}
+
+// File is a file/store with space allocation/deallocation support.
+type File struct {
+	f       storage.Accessor
+	atoms   int64       // current file size in atom units
+	canfree int64       // only blocks >= canfree can be subject to Free()
+	freetab [3857]int64 // freetab[0] is unused, freetab[1] is size 1 ptr, freetab[2] is size 2 ptr, ...
+	rwm     sync.RWMutex
+}
+
+func (f *File) read(b []byte, off int64) {
+	if n, err := f.f.ReadAt(b, off); n != len(b) {
+		panic(&ERead{f.f.Name(), off, err})
+	}
+}
+
+func (f *File) write(b []byte, off int64) {
+	if n, err := f.f.WriteAt(b, off); n != len(b) {
+		panic(&EWrite{f.f.Name(), off, err})
+	}
+}
+
+var ( // R/O
+	hdr   = []byte{0x0f, 0xf1, 0xc1, 0xa1, 0xfe, 0xa5, 0x1b, 0x1e, 0, 0, 0, 0, 0, 0, 2, 0} // free lists table @2
+	empty = make([]byte, 16)
+	zero  = []byte{0}
+	zero7 = make([]byte, 7)
+)
+
+// New returns a new File backed by store or an error if any.
+// Any existing data in store are discarded.
+func New(store storage.Accessor) (f *File, err error) {
+	f = &File{f: store}
+	return f, storage.Mutate(store, func() (err error) {
+		if err = f.f.Truncate(0); err != nil {
+			return &ECreate{f.f.Name(), err}
+		}
+
+		if _, err = f.Alloc(hdr[1:]); err != nil { //TODO internal panicking versions of the exported fns.
+			return
+		}
+
+		if _, err = f.Alloc(nil); err != nil { // (empty) root @1
+			return
+		}
+
+		b := make([]byte, 3856*14)
+		for i := 1; i <= 3856; i++ {
+			Handle(i).Put(b[(i-1)*14:])
+		}
+		if _, err = f.Alloc(b); err != nil {
+			return
+		}
+
+		f.canfree = f.atoms
+		return
+	})
+}
+
+// Open returns a new File backed by store or an error if any.
+// Store already has to be in a valid format.
+func Open(store storage.Accessor) (f *File, err error) {
+	defer func() {
+		if e := recover(); e != nil {
+			f = nil
+			err = e.(error)
+		}
+	}()
+
+	fi, err := store.Stat()
+	if err != nil {
+		panic(&EOpen{store.Name(), err})
+	}
+
+	fs := fi.Size()
+	if fs&0xf != 0 {
+		panic(&ESize{store.Name(), fi.Size()})
+	}
+
+	f = &File{f: store, atoms: fs >> 4}
+	b := make([]byte, len(hdr))
+	f.read(b, 0)
+	if !bytes.Equal(b, hdr) {
+		panic(&EHeader{store.Name(), b, append([]byte{}, hdr...)})
+	}
+
+	var atoms int64
+	b, atoms = f.readUsed(2)
+	f.canfree = atoms + 2
+	ofs := 0
+	var size, p Handle
+	for ofs < len(b) {
+		size.Get(b[ofs:])
+		ofs += 7
+		p.Get(b[ofs:])
+		ofs += 7
+		if sz, pp := int64(size), int64(p); size == 0 || size > 3856 || (pp != 0 && pp < f.canfree) || pp<<4 > fs-16 {
+			panic(&EFreeList{store.Name(), sz, pp})
+		}
+
+		f.freetab[size] = int64(p)
+	}
+	return
+}
+
+// Accessor returns the File's underlying Accessor.
+func (f *File) Accessor() storage.Accessor {
+	return f.f
+}
+
+// Close closes f and returns an error if any.
+func (f *File) Close() (err error) {
+	return storage.Mutate(f.Accessor(), func() (err error) {
+		if err = f.f.Close(); err != nil {
+			err = &EClose{f.f.Name(), err}
+		}
+		return
+	})
+}
+
+// Root returns the handle of the DB root (top level directory, ...).
+func (f *File) Root() Handle {
+	return 1
+}
+
+func (f *File) readUsed(atom int64) (content []byte, atoms int64) {
+	b, redirected := make([]byte, 7), false
+redir:
+	ofs := atom << 4
+	f.read(b[:1], ofs)
+	switch pre := b[0]; {
+	default:
+		panic(&ECorrupted{f.f.Name(), ofs})
+	case pre == 0x00: // Empty block
+	case pre >= 1 && pre <= 237: // Short
+		content = make([]byte, pre)
+		f.read(content, ofs+1)
+	case pre >= 0xee && pre <= 0xfb: // Short esc
+		content = make([]byte, 15+16*(pre-0xee))
+		f.read(content, ofs+1)
+		content[len(content)-1] += 0xfe
+	case pre == 0xfc: // Long
+		f.read(b[:2], ofs+1)
+		n := int(b[0])<<8 + int(b[1])
+		switch {
+		default:
+			panic(&ECorrupted{f.f.Name(), ofs + 1})
+		case n >= 238 && n <= 61680: // Long non esc
+			content = make([]byte, n)
+			f.read(content, ofs+3)
+		case n >= 61681: // Long esc
+			content = make([]byte, 13+16*(n-0xf0f1))
+			f.read(content, ofs+3)
+			content[len(content)-1] += 0xfe
+		}
+	case pre == 0xfd: // redir
+		if redirected {
+			panic(&ECorrupted{f.f.Name(), ofs})
+		}
+
+		f.read(b[:7], ofs+1)
+		(*Handle)(&atom).Get(b)
+		redirected = true
+		goto redir
+	}
+	return content, rq2Atoms(len(content))
+}
+
+func (f *File) writeUsed(b []byte, atom int64) {
+	n := len(b)
+	switch ofs, atoms, endmark := atom<<4, rq2Atoms(n), true; {
+	default:
+		panic("internal error")
+	case n == 0:
+		f.write(empty, ofs)
+	case n <= 237:
+		if (n+1)&0xf == 0 { // content end == atom end
+			if v := b[n-1]; v >= 0xfe { // escape
+				pre := []byte{byte((16*0xee + n - 15) >> 4)}
+				f.write(pre, ofs)
+				f.write(b[:n-1], ofs+1)
+				f.write([]byte{v - 0xfe}, ofs+atoms<<4-1)
+				return
+			}
+			endmark = false
+		}
+		// non esacpe
+		pre := []byte{byte(n)}
+		f.write(pre, ofs)
+		f.write(b, ofs+1)
+		if endmark {
+			f.write(zero, ofs+atoms<<4-1) // last block byte <- used block
+		}
+	case n > 237 && n <= 61680:
+		if (n+3)&0xf == 0 { // content end == atom end
+			if v := b[n-1]; v >= 0xfe { // escape
+				x := (16*0xf0f1 + n - 13) >> 4
+				pre := []byte{0xFC, byte(x >> 8), byte(x)}
+				f.write(pre, ofs)
+				f.write(b[:n-1], ofs+3)
+				f.write([]byte{v - 0xfe}, ofs+atoms<<4-1)
+				return
+			}
+			endmark = false
+		}
+		// non esacpe
+		pre := []byte{0xfc, byte(n >> 8), byte(n)}
+		f.write(pre, ofs)
+		f.write(b, ofs+3)
+		if endmark {
+			f.write(zero, ofs+atoms<<4-1) // last block byte <- used block
+		}
+	}
+}
+
+func rq2Atoms(rqbytes int) (rqatoms int64) {
+	if rqbytes > 237 {
+		rqbytes += 2
+	}
+	return int64(rqbytes>>4 + 1)
+}
+
+func (f *File) extend(b []byte) (handle int64) {
+	handle = f.atoms
+	f.writeUsed(b, handle)
+	f.atoms += rq2Atoms(len(b))
+	return
+}
+
+// Alloc stores b in a newly allocated space and returns its handle and an error if any.
+func (f *File) Alloc(b []byte) (handle Handle, err error) {
+	err = storage.Mutate(f.Accessor(), func() (err error) {
+		rqAtoms := rq2Atoms(len(b))
+		if rqAtoms > 3856 {
+			return &EBadRequest{f.f.Name(), len(b)}
+		}
+
+		for foundsize, foundp := range f.freetab[rqAtoms:] {
+			if foundp != 0 {
+				// this works only for the current unique sizes list (except the last item!)
+				size := int64(foundsize) + rqAtoms
+				handle = Handle(foundp)
+				if size == 3856 {
+					buf := make([]byte, 7)
+					f.read(buf, int64(handle)<<4+15)
+					(*Handle)(&size).Get(buf)
+				}
+				f.delFree(int64(handle), size)
+				if rqAtoms < size {
+					f.addFree(int64(handle)+rqAtoms, size-rqAtoms)
+				}
+				f.writeUsed(b, int64(handle))
+				return
+			}
+		}
+
+		handle = Handle(f.extend(b))
+		return
+	})
+	return
+}
+
+// checkLeft returns the atom size of a free bleck left adjacent to block @atom.
+// If that block is not free the returned size is 0.
+func (f *File) checkLeft(atom int64) (size int64) {
+	if atom <= f.canfree {
+		return
+	}
+
+	b := make([]byte, 7)
+	fp := atom << 4
+	f.read(b[:1], fp-1)
+	switch last := b[0]; {
+	case last <= 0xfd:
+		// used block
+	case last == 0xfe:
+		f.read(b, fp-8)
+		(*Handle)(&size).Get(b)
+	case last == 0xff:
+		size = 1
+	}
+	return
+}
+
+// getInfo returns the block @atom type and size.
+func (f *File) getInfo(atom int64) (pref byte, size int64) {
+	b := make([]byte, 7)
+	fp := atom << 4
+	f.read(b[:1], fp)
+	switch pref = b[0]; {
+	case pref == 0: // Empty used
+		size = 1
+	case pref >= 1 && pref <= 237: // Short
+		size = rq2Atoms(int(pref))
+	case pref >= 0xee && pref <= 0xfb: // Short esc
+		size = rq2Atoms(15 + 16*int(pref-0xee))
+	case pref == 0xfc: // Long
+		f.read(b[:2], fp+1)
+		n := int(b[0])<<8 + int(b[1])
+		switch {
+		default:
+			panic(&ECorrupted{f.f.Name(), fp + 1})
+		case n >= 238 && n <= 61680: // Long non esc
+			size = rq2Atoms(n)
+		case n >= 61681: // Long esc
+			size = rq2Atoms(13 + 16*(n-0xf0f1))
+		}
+	case pref == 0xfd: // reloc
+		size = 1
+	case pref == 0xfe:
+		f.read(b, fp+15)
+		(*Handle)(&size).Get(b)
+	case pref == 0xff:
+		size = 1
+	}
+	return
+}
+
+// getSize returns the atom size of the block @atom and wheter it is free.
+func (f *File) getSize(atom int64) (size int64, isFree bool) {
+	var typ byte
+	typ, size = f.getInfo(atom)
+	isFree = typ >= 0xfe
+	return
+}
+
+// checkRight returns the atom size of a free bleck right adjacent to block @atom,atoms.
+// If that block is not free the returned size is 0.
+func (f *File) checkRight(atom, atoms int64) (size int64) {
+	if atom+atoms >= f.atoms {
+		return
+	}
+
+	if sz, free := f.getSize(atom + atoms); free {
+		size = sz
+	}
+	return
+}
+
+// delFree removes the atoms@atom free block from the free block list
+func (f *File) delFree(atom, atoms int64) {
+	b := make([]byte, 15)
+	size := int(atoms)
+	if n := len(f.freetab); atoms >= int64(n) {
+		size = n - 1
+	}
+	fp := atom << 4
+	f.read(b[1:], fp+1)
+	var prev, next Handle
+	prev.Get(b[1:])
+	next.Get(b[8:])
+
+	switch {
+	case prev == 0 && next != 0:
+		next.Put(b)
+		f.write(b[:7], int64(32+3+7+(size-1)*14))
+		f.write(zero7, int64(next)<<4+1)
+		f.freetab[size] = int64(next)
+	case prev != 0 && next == 0:
+		f.write(zero7, int64(prev)<<4+8)
+	case prev != 0 && next != 0:
+		prev.Put(b)
+		f.write(b[:7], int64(next)<<4+1)
+		next.Put(b)
+		f.write(b[:7], int64(prev)<<4+8)
+	default: // prev == 0 && next == 0:
+		f.write(zero7, int64(32+3+7+(size-1)*14))
+		f.freetab[size] = 0
+	}
+}
+
+// addFree adds atoms@atom to the free block lists and marks it free.
+func (f *File) addFree(atom, atoms int64) {
+	b := make([]byte, 7)
+	size := int(atoms)
+	if n := len(f.freetab); atoms >= int64(n) {
+		size = n - 1
+	}
+	head := f.freetab[size]
+	if head == 0 { // empty list
+		f.makeFree(0, atom, atoms, 0)
+		Handle(atom).Put(b)
+		f.write(b, int64(32+3+7+(size-1)*14))
+		f.freetab[size] = atom
+		return
+	}
+
+	Handle(atom).Put(b)
+	f.write(b, head<<4+1)            // head.prev = atom
+	f.makeFree(0, atom, atoms, head) // atom.next = head
+	f.write(b, int64(32+3+7+(size-1)*14))
+	f.freetab[size] = atom
+}
+
+// makeFree sets up the content of a free block atoms@atom, fills the prev and next links.
+func (f *File) makeFree(prev, atom, atoms, next int64) {
+	b := make([]byte, 23)
+	fp := atom << 4
+	if atoms == 1 {
+		b[0] = 0xff
+		Handle(prev).Put(b[1:])
+		Handle(next).Put(b[8:])
+		b[15] = 0xff
+		f.write(b[:16], fp)
+		return
+	}
+
+	b[0] = 0xfe
+	Handle(prev).Put(b[1:])
+	Handle(next).Put(b[8:])
+	Handle(atoms).Put(b[15:])
+	f.write(b[:22], fp)
+	b[22] = 0xfe
+	f.write(b[15:], fp+atoms<<4-8)
+}
+
+// Read reads and return the data associated with handle and an error if any.
+// Passing an invalid handle to Read may return invalid data without error.
+// It's like getting garbage via passing an invalid pointer to C.memcopy().
+func (f *File) Read(handle Handle) (b []byte, err error) {
+	defer func() {
+		if e := recover(); e != nil {
+			b = nil
+			err = e.(error)
+		}
+	}()
+
+	switch handle {
+	case 0:
+		panic(ENullHandle(f.f.Name()))
+	case 2:
+		panic(&EHandle{f.f.Name(), handle})
+	default:
+		b, _ = f.readUsed(int64(handle))
+	}
+	return
+}
+
+// Free frees space associated with handle and returns an error if any. Passing an invalid
+// handle to Free or reusing handle afterwards will probably corrupt the database or provide
+// invalid data on Read. It's like corrupting memory via passing an invalid pointer to C.free()
+// or reusing that pointer.
+func (f *File) Free(handle Handle) (err error) {
+	return storage.Mutate(f.Accessor(), func() (err error) {
+		atom := int64(handle)
+		atoms, isFree := f.getSize(atom)
+		if isFree || atom < f.canfree {
+			return &EHandle{f.f.Name(), handle}
+		}
+
+		leftFree, rightFree := f.checkLeft(atom), f.checkRight(atom, atoms)
+		switch {
+		case leftFree != 0 && rightFree != 0:
+			f.delFree(atom-leftFree, leftFree)
+			f.delFree(atom+atoms, rightFree)
+			f.addFree(atom-leftFree, leftFree+atoms+rightFree)
+		case leftFree != 0 && rightFree == 0:
+			f.delFree(atom-leftFree, leftFree)
+			if atom+atoms == f.atoms { // the left free neighbour and this block together are an empy tail
+				f.atoms = atom - leftFree
+				f.f.Truncate(f.atoms << 4)
+				return
+			}
+
+			f.addFree(atom-leftFree, leftFree+atoms)
+		case leftFree == 0 && rightFree != 0:
+			f.delFree(atom+atoms, rightFree)
+			f.addFree(atom, atoms+rightFree)
+		default: // leftFree == 0 && rightFree == 0
+			if atom+atoms < f.atoms { // isolated inner block
+				f.addFree(atom, atoms)
+				return
+			}
+
+			f.f.Truncate(atom << 4) // isolated tail block, shrink file
+			f.atoms = atom
+		}
+		return
+	})
+}
+
+// Realloc reallocates space associted with handle to acomodate b, returns the newhandle
+// newly associated with b and an error if any. If keepHandle == true then Realloc guarantees
+// newhandle == handle even if the new data are larger then the previous content associated
+// with handle. If !keepHandle && newhandle != handle then reusing handle will probably corrupt
+// the database.
+// The above effects are like corrupting memory/data via passing an invalid pointer to C.realloc().
+func (f *File) Realloc(handle Handle, b []byte, keepHandle bool) (newhandle Handle, err error) {
+	err = storage.Mutate(f.Accessor(), func() (err error) {
+		switch handle {
+		case 0, 2:
+			return &EHandle{f.f.Name(), handle}
+		case 1:
+			keepHandle = true
+		}
+		newhandle = handle
+		atom, newatoms := int64(handle), rq2Atoms(len(b))
+		if newatoms > 3856 {
+			return &EBadRequest{f.f.Name(), len(b)}
+		}
+
+		typ, oldatoms := f.getInfo(atom)
+		switch {
+		default:
+			return &ECorrupted{f.f.Name(), atom << 4}
+		case typ <= 0xfc: // non relocated used block
+			switch {
+			case newatoms == oldatoms: // in place replace
+				f.writeUsed(b, atom)
+			case newatoms < oldatoms: // in place shrink
+				rightFree := f.checkRight(atom, oldatoms)
+				if rightFree > 0 { // right join
+					f.delFree(atom+oldatoms, rightFree)
+				}
+				f.addFree(atom+newatoms, oldatoms+rightFree-newatoms)
+				f.writeUsed(b, atom)
+			case newatoms > oldatoms:
+				if rightFree := f.checkRight(atom, oldatoms); rightFree > 0 && newatoms <= oldatoms+rightFree {
+					f.delFree(atom+oldatoms, rightFree)
+					if newatoms < oldatoms+rightFree {
+						f.addFree(atom+newatoms, oldatoms+rightFree-newatoms)
+					}
+					f.writeUsed(b, atom)
+					return
+				}
+
+				if !keepHandle {
+					f.Free(Handle(atom))
+					newhandle, err = f.Alloc(b)
+					return
+				}
+
+				// reloc
+				newatom, e := f.Alloc(b)
+				if e != nil {
+					return e
+				}
+
+				buf := make([]byte, 16)
+				buf[0] = 0xfd
+				Handle(newatom).Put(buf[1:])
+				f.Realloc(Handle(atom), buf[1:], true)
+				f.write(buf[:1], atom<<4)
+			}
+		case typ == 0xfd: // reloc
+			var target Handle
+			buf := make([]byte, 7)
+			f.read(buf, atom<<4+1)
+			target.Get(buf)
+			switch {
+			case newatoms == 1:
+				f.writeUsed(b, atom)
+				f.Free(target)
+			default:
+				if rightFree := f.checkRight(atom, 1); rightFree > 0 && newatoms <= 1+rightFree {
+					f.delFree(atom+1, rightFree)
+					if newatoms < 1+rightFree {
+						f.addFree(atom+newatoms, 1+rightFree-newatoms)
+					}
+					f.writeUsed(b, atom)
+					f.Free(target)
+					return
+				}
+
+				newtarget, e := f.Realloc(Handle(target), b, false)
+				if e != nil {
+					return e
+				}
+
+				if newtarget != target {
+					Handle(newtarget).Put(buf)
+					f.write(buf, atom<<4+1)
+				}
+			}
+		}
+		return
+	})
+	return
+}
+
+// Lock locks f for writing. If the lock is already locked for reading or writing,
+// Lock blocks until the lock is available. To ensure that the lock eventually becomes available,
+// a blocked Lock call excludes new readers from acquiring the lock.
+func (f *File) Lock() {
+	f.rwm.Lock()
+}
+
+// RLock locks f for reading. If the lock is already locked for writing or there is a writer
+// already waiting to release the lock, RLock blocks until the writer has released the lock.
+func (f *File) RLock() {
+	f.rwm.RLock()
+}
+
+// Unlock unlocks f for writing. It is a run-time error if f is not locked for writing on entry to Unlock.
+//
+// As with Mutexes, a locked RWMutex is not associated with a particular goroutine.
+// One goroutine may RLock (Lock) f and then arrange for another goroutine to RUnlock (Unlock) it.
+func (f *File) Unlock() {
+	f.rwm.Unlock()
+}
+
+// RUnlock undoes a single RLock call; it does not affect other simultaneous readers.
+// It is a run-time error if f is not locked for reading on entry to RUnlock.
+func (f *File) RUnlock() {
+	f.rwm.RUnlock()
+}
+
+// LockedAlloc wraps Alloc in a Lock/Unlock pair.
+func (f *File) LockedAlloc(b []byte) (handle Handle, err error) {
+	f.Lock()
+	defer f.Unlock()
+	return f.Alloc(b)
+}
+
+// LockedFree wraps Free in a Lock/Unlock pair.
+func (f *File) LockedFree(handle Handle) (err error) {
+	f.Lock()
+	defer f.Unlock()
+	return f.Free(handle)
+}
+
+// LockedRead wraps Read in a RLock/RUnlock pair.
+func (f *File) LockedRead(handle Handle) (b []byte, err error) {
+	f.RLock()
+	defer f.RUnlock()
+	return f.Read(handle)
+}
+
+// LockedRealloc wraps Realloc in a Lock/Unlock pair.
+func (f *File) LockedRealloc(handle Handle, b []byte, keepHandle bool) (newhandle Handle, err error) {
+	f.Lock()
+	defer f.Unlock()
+	return f.Realloc(handle, b, keepHandle)
+}

Godeps/_workspace/src/github.com/cznic/fileutil/falloc/test_deps.go

@@ -0,0 +1,16 @@
+// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// blame: jnml, labs.nic.cz
+
+package falloc
+
+// Pull test dependencies too.
+// Enables easy 'go test X' after 'go get X'
+import (
+	_ "github.com/coreos/rocket/Godeps/_workspace/src/github.com/cznic/fileutil"
+	_ "github.com/coreos/rocket/Godeps/_workspace/src/github.com/cznic/fileutil/storage"
+
+	_ "github.com/coreos/rocket/Godeps/_workspace/src/github.com/cznic/mathutil"
+)

Godeps/_workspace/src/github.com/cznic/fileutil/fileutil.go

@@ -0,0 +1,223 @@
+// Copyright (c) 2014 The fileutil Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package fileutil collects some file utility functions.
+package fileutil
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"sync"
+	"time"
+)
+
+// GoMFile is a concurrent access safe version of MFile.
+type GoMFile struct {
+	mfile *MFile
+	mutex sync.Mutex
+}
+
+// NewGoMFile return a newly created GoMFile.
+func NewGoMFile(fname string, flag int, perm os.FileMode, delta_ns int64) (m *GoMFile, err error) {
+	m = &GoMFile{}
+	if m.mfile, err = NewMFile(fname, flag, perm, delta_ns); err != nil {
+		m = nil
+	}
+	return
+}
+
+func (m *GoMFile) File() (file *os.File, err error) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+	return m.mfile.File()
+}
+
+func (m *GoMFile) SetChanged() {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+	m.mfile.SetChanged()
+}
+
+func (m *GoMFile) SetHandler(h MFileHandler) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+	m.mfile.SetHandler(h)
+}
+
+// MFileHandler resolves modifications of File.
+// Possible File context is expected to be a part of the handler's closure.
+type MFileHandler func(*os.File) error
+
+// MFile represents an os.File with a guard/handler on change/modification.
+// Example use case is an app with a configuration file which can be modified at any time
+// and have to be reloaded in such event prior to performing something configurable by that
+// file. The checks are made only on access to the MFile file by
+// File() and a time threshold/hysteresis value can be chosen on creating a new MFile.
+type MFile struct {
+	file    *os.File
+	handler MFileHandler
+	t0      int64
+	delta   int64
+	ctime   int64
+}
+
+// NewMFile returns a newly created MFile or Error if any.
+// The fname, flag and perm parameters have the same meaning as in os.Open.
+// For meaning of the delta_ns parameter please see the (m *MFile) File() docs.
+func NewMFile(fname string, flag int, perm os.FileMode, delta_ns int64) (m *MFile, err error) {
+	m = &MFile{}
+	m.t0 = time.Now().UnixNano()
+	if m.file, err = os.OpenFile(fname, flag, perm); err != nil {
+		return
+	}
+
+	var fi os.FileInfo
+	if fi, err = m.file.Stat(); err != nil {
+		return
+	}
+
+	m.ctime = fi.ModTime().UnixNano()
+	m.delta = delta_ns
+	runtime.SetFinalizer(m, func(m *MFile) {
+		m.file.Close()
+	})
+	return
+}
+
+// SetChanged forces next File() to unconditionally handle modification of the wrapped os.File.
+func (m *MFile) SetChanged() {
+	m.ctime = -1
+}
+
+// SetHandler sets a function to be invoked when modification of MFile is to be processed.
+func (m *MFile) SetHandler(h MFileHandler) {
+	m.handler = h
+}
+
+// File returns an os.File from MFile. If time elapsed between the last invocation of this function
+// and now is at least delta_ns ns (a parameter of NewMFile) then the file is checked for
+// change/modification. For delta_ns == 0 the modification is checked w/o getting os.Time().
+// If a change is detected a handler is invoked on the MFile file.
+// Any of these steps can produce an Error. If that happens the function returns nil, Error.
+func (m *MFile) File() (file *os.File, err error) {
+	var now int64
+
+	mustCheck := m.delta == 0
+	if !mustCheck {
+		now = time.Now().UnixNano()
+		mustCheck = now-m.t0 > m.delta
+	}
+
+	if mustCheck { // check interval reached
+		var fi os.FileInfo
+		if fi, err = m.file.Stat(); err != nil {
+			return
+		}
+
+		if fi.ModTime().UnixNano() != m.ctime { // modification detected
+			if m.handler == nil {
+				return nil, fmt.Errorf("no handler set for modified file %q", m.file.Name())
+			}
+			if err = m.handler(m.file); err != nil {
+				return
+			}
+
+			m.ctime = fi.ModTime().UnixNano()
+		}
+		m.t0 = now
+	}
+
+	return m.file, nil
+}
+
+// Read reads buf from r. It will either fill the full buf or fail.
+// It wraps the functionality of an io.Reader which may return less bytes than requested,
+// but may block if not all data are ready for the io.Reader.
+func Read(r io.Reader, buf []byte) (err error) {
+	have := 0
+	remain := len(buf)
+	got := 0
+	for remain > 0 {
+		if got, err = r.Read(buf[have:]); err != nil {
+			return
+		}
+
+		remain -= got
+		have += got
+	}
+	return
+}
+
+// "os" and/or "syscall" extensions
+
+// FadviseAdvice is used by Fadvise.
+type FadviseAdvice int
+
+// FAdviseAdvice values.
+const (
+	// $ grep FADV /usr/include/bits/fcntl.h
+	POSIX_FADV_NORMAL     FadviseAdvice = iota // No further special treatment.
+	POSIX_FADV_RANDOM                          // Expect random page references.
+	POSIX_FADV_SEQUENTIAL                      // Expect sequential page references.
+	POSIX_FADV_WILLNEED                        // Will need these pages.
+	POSIX_FADV_DONTNEED                        // Don't need these pages.
+	POSIX_FADV_NOREUSE                         // Data will be accessed once.
+)
+
+// TempFile creates a new temporary file in the directory dir with a name
+// ending with suffix, basename starting with prefix, opens the file for
+// reading and writing, and returns the resulting *os.File.  If dir is the
+// empty string, TempFile uses the default directory for temporary files (see
+// os.TempDir).  Multiple programs calling TempFile simultaneously will not
+// choose the same file.  The caller can use f.Name() to find the pathname of
+// the file.  It is the caller's responsibility to remove the file when no
+// longer needed.
+//
+// NOTE: This function differs from ioutil.TempFile.
+func TempFile(dir, prefix, suffix string) (f *os.File, err error) {
+	if dir == "" {
+		dir = os.TempDir()
+	}
+
+	nconflict := 0
+	for i := 0; i < 10000; i++ {
+		name := filepath.Join(dir, prefix+nextInfix()+suffix)
+		f, err = os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600)
+		if os.IsExist(err) {
+			if nconflict++; nconflict > 10 {
+				rand = reseed()
+			}
+			continue
+		}
+		break
+	}
+	return
+}
+
+// Random number state.
+// We generate random temporary file names so that there's a good
+// chance the file doesn't exist yet - keeps the number of tries in
+// TempFile to a minimum.
+var rand uint32
+var randmu sync.Mutex
+
+func reseed() uint32 {
+	return uint32(time.Now().UnixNano() + int64(os.Getpid()))
+}
+
+func nextInfix() string {
+	randmu.Lock()
+	r := rand
+	if r == 0 {
+		r = reseed()
+	}
+	r = r*1664525 + 1013904223 // constants from Numerical Recipes
+	rand = r
+	randmu.Unlock()
+	return strconv.Itoa(int(1e9 + r%1e9))[1:]
+}

Godeps/_workspace/src/github.com/cznic/fileutil/fileutil_arm.go

@@ -0,0 +1,25 @@
+// Copyright (c) 2014 The fileutil Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package fileutil
+
+import (
+	"io"
+	"os"
+)
+
+// PunchHole deallocates space inside a file in the byte range starting at
+// offset and continuing for len bytes. Not supported on ARM.
+func PunchHole(f *os.File, off, len int64) error {
+	return nil
+}
+
+// Fadvise predeclares an access pattern for file data.  See also 'man 2
+// posix_fadvise'. Not supported on ARM.
+func Fadvise(f *os.File, off, len int64, advice FadviseAdvice) error {
+	return nil
+}
+
+// IsEOF reports whether err is an EOF condition.
+func IsEOF(err error) bool { return err == io.EOF }

Godeps/_workspace/src/github.com/cznic/fileutil/fileutil_darwin.go

@@ -0,0 +1,25 @@
+// Copyright (c) 2014 The fileutil Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package fileutil
+
+import (
+	"io"
+	"os"
+)
+
+// PunchHole deallocates space inside a file in the byte range starting at
+// offset and continuing for len bytes. Not supported on OSX.
+func PunchHole(f *os.File, off, len int64) error {
+	return nil
+}
+
+// Fadvise predeclares an access pattern for file data.  See also 'man 2
+// posix_fadvise'. Not supported on OSX.
+func Fadvise(f *os.File, off, len int64, advice FadviseAdvice) error {
+	return nil
+}
+
+// IsEOF reports whether err is an EOF condition.
+func IsEOF(err error) bool { return err == io.EOF }

Godeps/_workspace/src/github.com/cznic/fileutil/fileutil_freebsd.go

@@ -0,0 +1,25 @@
+// Copyright (c) 2014 The fileutil Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package fileutil
+
+import (
+	"io"
+	"os"
+)
+
+// PunchHole deallocates space inside a file in the byte range starting at
+// offset and continuing for len bytes. Unimplemented on FreeBSD.
+func PunchHole(f *os.File, off, len int64) error {
+	return nil
+}
+
+// Fadvise predeclares an access pattern for file data.  See also 'man 2
+// posix_fadvise'. Unimplemented on FreeBSD.
+func Fadvise(f *os.File, off, len int64, advice FadviseAdvice) error {
+	return nil
+}
+
+// IsEOF reports whether err is an EOF condition.
+func IsEOF(err error) bool { return err == io.EOF }

Godeps/_workspace/src/github.com/cznic/fileutil/fileutil_linux.go

@@ -0,0 +1,96 @@
+// Copyright (c) 2014 The fileutil Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !arm
+
+package fileutil
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+	"os"
+	"strconv"
+	"syscall"
+)
+
+func n(s []byte) byte {
+	for i, c := range s {
+		if c < '0' || c > '9' {
+			s = s[:i]
+			break
+		}
+	}
+	v, _ := strconv.Atoi(string(s))
+	return byte(v)
+}
+
+func init() {
+	b, err := ioutil.ReadFile("/proc/sys/kernel/osrelease")
+	if err != nil {
+		panic(err)
+	}
+
+	tokens := bytes.Split(b, []byte("."))
+	if len(tokens) > 3 {
+		tokens = tokens[:3]
+	}
+	switch len(tokens) {
+	case 3:
+		// Supported since kernel 2.6.38
+		if bytes.Compare([]byte{n(tokens[0]), n(tokens[1]), n(tokens[2])}, []byte{2, 6, 38}) < 0 {
+			puncher = func(*os.File, int64, int64) error { return nil }
+		}
+	case 2:
+		if bytes.Compare([]byte{n(tokens[0]), n(tokens[1])}, []byte{2, 7}) < 0 {
+			puncher = func(*os.File, int64, int64) error { return nil }
+		}
+	default:
+		puncher = func(*os.File, int64, int64) error { return nil }
+	}
+}
+
+var puncher = func(f *os.File, off, len int64) error {
+	const (
+		/*
+			/usr/include/linux$ grep FL_ falloc.h
+		*/
+		_FALLOC_FL_KEEP_SIZE  = 0x01 // default is extend size
+		_FALLOC_FL_PUNCH_HOLE = 0x02 // de-allocates range
+	)
+
+	_, _, errno := syscall.Syscall6(
+		syscall.SYS_FALLOCATE,
+		uintptr(f.Fd()),
+		uintptr(_FALLOC_FL_KEEP_SIZE|_FALLOC_FL_PUNCH_HOLE),
+		uintptr(off),
+		uintptr(len),
+		0, 0)
+	if errno != 0 {
+		return os.NewSyscallError("SYS_FALLOCATE", errno)
+	}
+	return nil
+}
+
+// PunchHole deallocates space inside a file in the byte range starting at
+// offset and continuing for len bytes. No-op for kernels < 2.6.38 (or < 2.7).
+func PunchHole(f *os.File, off, len int64) error {
+	return puncher(f, off, len)
+}
+
+// Fadvise predeclares an access pattern for file data.  See also 'man 2
+// posix_fadvise'.
+func Fadvise(f *os.File, off, len int64, advice FadviseAdvice) error {
+	_, _, errno := syscall.Syscall6(
+		syscall.SYS_FADVISE64,
+		uintptr(f.Fd()),
+		uintptr(off),
+		uintptr(len),
+		uintptr(advice),
+		0, 0)
+	return os.NewSyscallError("SYS_FADVISE64", errno)
+}
+
+// IsEOF reports whether err is an EOF condition.
+func IsEOF(err error) bool { return err == io.EOF }

Godeps/_workspace/src/github.com/cznic/fileutil/fileutil_openbsd.go

@@ -0,0 +1,25 @@
+// Copyright (c) 2014 The fileutil Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package fileutil
+
+import (
+	"io"
+	"os"
+)
+
+// PunchHole deallocates space inside a file in the byte range starting at
+// offset and continuing for len bytes. Similar to FreeBSD, this is
+// unimplemented.
+func PunchHole(f *os.File, off, len int64) error {
+	return nil
+}
+
+// Unimplemented on OpenBSD.
+func Fadvise(f *os.File, off, len int64, advice FadviseAdvice) error {
+	return nil
+}
+
+// IsEOF reports whether err is an EOF condition.
+func IsEOF(err error) bool { return err == io.EOF }

Godeps/_workspace/src/github.com/cznic/fileutil/fileutil_plan9.go

@@ -0,0 +1,25 @@
+// Copyright (c) 2014 The fileutil Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package fileutil
+
+import (
+	"io"
+	"os"
+)
+
+// PunchHole deallocates space inside a file in the byte range starting at
+// offset and continuing for len bytes. Unimplemented on Plan 9.
+func PunchHole(f *os.File, off, len int64) error {
+	return nil
+}
+
+// Fadvise predeclares an access pattern for file data.  See also 'man 2
+// posix_fadvise'. Unimplemented on Plan 9.
+func Fadvise(f *os.File, off, len int64, advice FadviseAdvice) error {
+	return nil
+}
+
+// IsEOF reports whether err is an EOF condition.
+func IsEOF(err error) bool { return err == io.EOF }

Godeps/_workspace/src/github.com/cznic/fileutil/fileutil_solaris.go

@@ -0,0 +1,27 @@
+// Copyright (c) 2013 jnml. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build go1.3
+
+package fileutil
+
+import (
+	"io"
+	"os"
+)
+
+// PunchHole deallocates space inside a file in the byte range starting at
+// offset and continuing for len bytes. Not supported on Solaris.
+func PunchHole(f *os.File, off, len int64) error {
+	return nil
+}
+
+// Fadvise predeclares an access pattern for file data.  See also 'man 2
+// posix_fadvise'. Not supported on Solaris.
+func Fadvise(f *os.File, off, len int64, advice FadviseAdvice) error {
+	return nil
+}
+
+// IsEOF reports whether err is an EOF condition.
+func IsEOF(err error) bool { return err == io.EOF }

Godeps/_workspace/src/github.com/cznic/fileutil/fileutil_windows.go

@@ -0,0 +1,183 @@
+// Copyright (c) 2014 The fileutil Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package fileutil
+
+import (
+	"io"
+	"os"
+	"sync"
+	"syscall"
+	"unsafe"
+)
+
+// PunchHole deallocates space inside a file in the byte range starting at
+// offset and continuing for len bytes. Not supported on Windows.
+func PunchHole(f *os.File, off, len int64) error {
+	return puncher(f, off, len)
+}
+
+// Fadvise predeclares an access pattern for file data.  See also 'man 2
+// posix_fadvise'. Not supported on Windows.
+func Fadvise(f *os.File, off, len int64, advice FadviseAdvice) error {
+	return nil
+}
+
+// IsEOF reports whether err is an EOF condition.
+func IsEOF(err error) bool {
+	if err == io.EOF {
+		return true
+	}
+
+	// http://social.technet.microsoft.com/Forums/windowsserver/en-US/1a16311b-c625-46cf-830b-6a26af488435/how-to-solve-error-38-0x26-errorhandleeof-using-fsctlgetretrievalpointers
+	x, ok := err.(*os.PathError)
+	return ok && x.Op == "read" && x.Err.(syscall.Errno) == 0x26
+}
+
+var (
+	modkernel32 = syscall.NewLazyDLL("kernel32.dll")
+
+	procDeviceIOControl = modkernel32.NewProc("DeviceIoControl")
+
+	sparseFilesMu sync.Mutex
+	sparseFiles   map[uintptr]struct{}
+)
+
+func init() {
+	// sparseFiles is an fd set for already "sparsed" files - according to
+	// msdn.microsoft.com/en-us/library/windows/desktop/aa364225(v=vs.85).aspx
+	// the file handles are unique per process.
+	sparseFiles = make(map[uintptr]struct{})
+}
+
+// puncHoleWindows punches a hole into the given file starting at offset,
+// measuring "size" bytes
+// (http://msdn.microsoft.com/en-us/library/windows/desktop/aa364597%28v=vs.85%29.aspx)
+func puncher(file *os.File, offset, size int64) error {
+	if err := ensureFileSparse(file); err != nil {
+		return err
+	}
+
+	// http://msdn.microsoft.com/en-us/library/windows/desktop/aa364411%28v=vs.85%29.aspx
+	// typedef struct _FILE_ZERO_DATA_INFORMATION {
+	//  LARGE_INTEGER FileOffset;
+	//  LARGE_INTEGER BeyondFinalZero;
+	//} FILE_ZERO_DATA_INFORMATION, *PFILE_ZERO_DATA_INFORMATION;
+	type fileZeroDataInformation struct {
+		FileOffset, BeyondFinalZero int64
+	}
+
+	lpInBuffer := fileZeroDataInformation{
+		FileOffset:      offset,
+		BeyondFinalZero: offset + size}
+	return deviceIOControl(false, file.Fd(), uintptr(unsafe.Pointer(&lpInBuffer)), 16)
+}
+
+// // http://msdn.microsoft.com/en-us/library/windows/desktop/cc948908%28v=vs.85%29.aspx
+// type fileSetSparseBuffer struct {
+//	 SetSparse bool
+// }
+
+func ensureFileSparse(file *os.File) (err error) {
+	fd := file.Fd()
+	sparseFilesMu.Lock()
+	if _, ok := sparseFiles[fd]; ok {
+		sparseFilesMu.Unlock()
+		return nil
+	}
+
+	if err = deviceIOControl(true, fd, 0, 0); err == nil {
+		sparseFiles[fd] = struct{}{}
+	}
+	sparseFilesMu.Unlock()
+	return err
+}
+
+func deviceIOControl(setSparse bool, fd, inBuf, inBufLen uintptr) (err error) {
+	const (
+		//http://source.winehq.org/source/include/winnt.h#L4605
+		file_read_data  = 1
+		file_write_data = 2
+
+		// METHOD_BUFFERED	0
+		method_buffered = 0
+		// FILE_ANY_ACCESS   0
+		file_any_access = 0
+		// FILE_DEVICE_FILE_SYSTEM   0x00000009
+		file_device_file_system = 0x00000009
+		// FILE_SPECIAL_ACCESS   (FILE_ANY_ACCESS)
+		file_special_access = file_any_access
+		file_read_access    = file_read_data
+		file_write_access   = file_write_data
+
+		// http://source.winehq.org/source/include/winioctl.h
+		// #define CTL_CODE 	(  	DeviceType,
+		//		Function,
+		//		Method,
+		//		Access  		 )
+		//    ((DeviceType) << 16) | ((Access) << 14) | ((Function) << 2) | (Method)
+
+		// FSCTL_SET_COMPRESSION   CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 16, METHOD_BUFFERED, FILE_READ_DATA | FILE_WRITE_DATA)
+		fsctl_set_compression = (file_device_file_system << 16) | ((file_read_access | file_write_access) << 14) | (16 << 2) | method_buffered
+		// FSCTL_SET_SPARSE   CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 49, METHOD_BUFFERED, FILE_SPECIAL_ACCESS)
+		fsctl_set_sparse = (file_device_file_system << 16) | (file_special_access << 14) | (49 << 2) | method_buffered
+		// FSCTL_SET_ZERO_DATA   CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 50, METHOD_BUFFERED, FILE_WRITE_DATA)
+		fsctl_set_zero_data = (file_device_file_system << 16) | (file_write_data << 14) | (50 << 2) | method_buffered
+	)
+	retPtr := uintptr(unsafe.Pointer(&(make([]byte, 8)[0])))
+	var r1 uintptr
+	var e1 syscall.Errno
+	if setSparse {
+		// BOOL
+		// WINAPI
+		// DeviceIoControl( (HANDLE) hDevice,                      // handle to a file
+		//                  FSCTL_SET_SPARSE,                      // dwIoControlCode
+		//                  (PFILE_SET_SPARSE_BUFFER) lpInBuffer,  // input buffer
+		//                  (DWORD) nInBufferSize,                 // size of input buffer
+		//                  NULL,                                  // lpOutBuffer
+		//                  0,                                     // nOutBufferSize
+		//                  (LPDWORD) lpBytesReturned,             // number of bytes returned
+		//                  (LPOVERLAPPED) lpOverlapped );         // OVERLAPPED structure
+		r1, _, e1 = syscall.Syscall9(procDeviceIOControl.Addr(), 8,
+			fd,
+			uintptr(fsctl_set_sparse),
+			// If the lpInBuffer parameter is NULL, the operation will behave the same as if the SetSparse member of the FILE_SET_SPARSE_BUFFER structure were TRUE. In other words, the operation sets the file to a sparse file.
+			0, // uintptr(unsafe.Pointer(&lpInBuffer)),
+			0, // 1,
+			0,
+			0,
+			retPtr,
+			0,
+			0)
+	} else {
+		// BOOL
+		// WINAPI
+		// DeviceIoControl( (HANDLE) hDevice,              // handle to a file
+		//                  FSCTL_SET_ZERO_DATA,           // dwIoControlCode
+		//                  (LPVOID) lpInBuffer,           // input buffer
+		//                  (DWORD) nInBufferSize,         // size of input buffer
+		//                  NULL,                          // lpOutBuffer
+		//                  0,                             // nOutBufferSize
+		//                  (LPDWORD) lpBytesReturned,     // number of bytes returned
+		//                  (LPOVERLAPPED) lpOverlapped ); // OVERLAPPED structure
+		r1, _, e1 = syscall.Syscall9(procDeviceIOControl.Addr(), 8,
+			fd,
+			uintptr(fsctl_set_zero_data),
+			inBuf,
+			inBufLen,
+			0,
+			0,
+			retPtr,
+			0,
+			0)
+	}
+	if r1 == 0 {
+		if e1 != 0 {
+			err = error(e1)
+		} else {
+			err = syscall.EINVAL
+		}
+	}
+	return err
+}

Godeps/_workspace/src/github.com/cznic/fileutil/hdb/LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of CZ.NIC nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

Godeps/_workspace/src/github.com/cznic/fileutil/hdb/README

@@ -0,0 +1,5 @@
+This is a goinstall-able mirror of modified code already published at:
+https://git.nic.cz/redmine/projects/gofileutil/repository/show/hdb
+
+Install: $go get github.com/cznic/fileutil/hdb
+Godocs: http://gopkgdoc.appspot.com/pkg/github.com/cznic/fileutil/hdb

Godeps/_workspace/src/github.com/cznic/fileutil/hdb/all_test.go

@@ -0,0 +1,15 @@
+// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// blame: jnml, labs.nic.cz
+
+package hdb
+
+import (
+	"testing"
+)
+
+func TestPlaceholder(t *testing.T) {
+	t.Log("TODO") //TODO
+}

Godeps/_workspace/src/github.com/cznic/fileutil/hdb/hdb.go

@@ -0,0 +1,153 @@
+// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// blame: jnml, labs.nic.cz
+
+/*
+WIP: Package hdb provides a "handle"/value DB like store, but actually it's
+closer to the model of a process's virtual memory and its alloc, free and move
+methods.
+
+The hdb package is a thin layer around falloc.File providing stable-only
+handles and the basic synchronizing primitives.  The central functionality of
+hdb are the New, Set, Get and Delete methods of Store.
+
+Conceptual analogy:
+	New	    alloc(sizeof(content)), return new "memory" pointer (a handle).
+
+	Get	    memmove() from "memory" "pointed to" by handle to the result content.
+		    Note: Handle "knows" the size of its content.
+
+	Set	    memmove() from content to "memory" pointed to by handle.
+		    In contrast to real memory, the new content may have different
+		    size than the previously stored one w/o additional handling
+		    and the "pointer" handle remains the same.
+
+	Delete	    free() the "memory" "pointed to" by handle.
+*/
+package hdb
+
+import (
+	"github.com/coreos/rocket/Godeps/_workspace/src/github.com/cznic/fileutil/falloc"
+	"github.com/coreos/rocket/Godeps/_workspace/src/github.com/cznic/fileutil/storage"
+)
+
+type Store struct {
+	f *falloc.File
+}
+
+// New returns a newly created Store backed by accessor, discarding its conents if any.
+// If successful, methods on the returned Store can be used for I/O.
+// It returns the Store and an error, if any.
+func New(accessor storage.Accessor) (store *Store, err error) {
+	s := &Store{}
+	if s.f, err = falloc.New(accessor); err == nil {
+		store = s
+	}
+	return
+}
+
+// Open opens the Store from accessor.
+// If successful, methods on the returned Store can be used for data exchange.
+// It returns the Store and an error, if any.
+func Open(accessor storage.Accessor) (store *Store, err error) {
+	s := &Store{}
+	if s.f, err = falloc.Open(accessor); err == nil {
+		store = s
+	}
+	return
+}
+
+// Close closes the store. Further access to the store has undefined behavior and may panic.
+// It returns an error, if any.
+func (s *Store) Close() (err error) {
+	defer func() {
+		s.f = nil
+	}()
+
+	return s.f.Close()
+}
+
+// Delete deletes the data associated with handle.
+// It returns an error if any.
+func (s *Store) Delete(handle falloc.Handle) (err error) {
+	return s.f.Free(handle)
+}
+
+// Get gets the data associated with handle.
+// It returns the data and an error, if any.
+func (s *Store) Get(handle falloc.Handle) (b []byte, err error) {
+	return s.f.Read(handle)
+}
+
+// New associates data with a new handle.
+// It returns the handle and an error, if any.
+func (s *Store) New(b []byte) (handle falloc.Handle, err error) {
+	return s.f.Alloc(b)
+}
+
+// Set associates data with an existing handle.
+// It returns an error, if any.
+func (s *Store) Set(handle falloc.Handle, b []byte) (err error) {
+	_, err = s.f.Realloc(handle, b, true)
+	return
+}
+
+// Root returns the handle of the DB root (top level directory, ...).
+func (s *Store) Root() falloc.Handle {
+	return s.f.Root()
+}
+
+// File returns the underlying falloc.File of 's'.
+func (s *Store) File() *falloc.File {
+	return s.f
+}
+
+// Lock locks 's' for writing. If the lock is already locked for reading or writing,
+// Lock blocks until the lock is available. To ensure that the lock eventually becomes available,
+// a blocked Lock call excludes new readers from acquiring the lock.
+func (s *Store) Lock() {
+	s.f.Lock()
+}
+
+// RLock locks 's' for reading. If the lock is already locked for writing or there is a writer
+// already waiting to release the lock, RLock blocks until the writer has released the lock.
+func (s *Store) RLock() {
+	s.f.RLock()
+}
+
+// Unlock unlocks 's' for writing. It's a run-time error if 's' is not locked for writing on entry to Unlock.
+//
+// As with Mutexes, a locked RWMutex is not associated with a particular goroutine.
+// One goroutine may RLock (Lock) 's' and then arrange for another goroutine to RUnlock (Unlock) it.
+func (s *Store) Unlock() {
+	s.f.Unlock()
+}
+
+// RUnlock undoes a single RLock call; it does not affect other simultaneous readers.
+// It's a run-time error if 's' is not locked for reading on entry to RUnlock.
+func (s *Store) RUnlock() {
+	s.f.RUnlock()
+}
+
+// LockedNew wraps New in a Lock/Unlock pair.
+func (s *Store) LockedNew(b []byte) (handle falloc.Handle, err error) {
+	return s.f.LockedAlloc(b)
+}
+
+// LockedDelete wraps Delete in a Lock/Unlock pair.
+func (s *Store) LockedDelete(handle falloc.Handle) (err error) {
+	return s.f.LockedFree(handle)
+}
+
+// LockedGet wraps Get in a RLock/RUnlock pair.
+func (s *Store) LockedGet(handle falloc.Handle) (b []byte, err error) {
+	return s.f.LockedRead(handle)
+}
+
+// LockedSet wraps Set in a Lock/Unlock pair.
+func (s *Store) LockedSet(handle falloc.Handle, b []byte) (err error) {
+	_, err = s.f.Realloc(handle, b, true)
+	return
+}

Godeps/_workspace/src/github.com/cznic/fileutil/hdb/test_deps.go

@@ -0,0 +1,13 @@
+// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// blame: jnml, labs.nic.cz
+
+package hdb
+
+// Pull test dependencies too.
+// Enables easy 'go test X' after 'go get X'
+import (
+// nothing yet
+)

Godeps/_workspace/src/github.com/cznic/fileutil/punch_test.go

@@ -0,0 +1,55 @@
+// Copyright (c) 2014 The fileutil authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package fileutil
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+func TestPunch(t *testing.T) {
+	file, err := ioutil.TempFile("", "punchhole-")
+	if err != nil {
+		t.Error(err)
+	}
+	defer os.Remove(file.Name())
+	defer file.Close()
+	buf := make([]byte, 10<<20)
+	for i := range buf {
+		buf[i] = byte(1 + (i+1)&0xfe)
+	}
+	if _, err = file.Write(buf); err != nil {
+		t.Errorf("error writing to the temp file: %v", err)
+		t.FailNow()
+	}
+	if err = file.Sync(); err != nil {
+		t.Logf("error syncing %q: %v", file.Name(), err)
+	}
+	for i, j := range []int{1, 31, 1 << 10} {
+		if err = PunchHole(file, int64(j), int64(j)); err != nil {
+			t.Errorf("%d. error punching at %d, size %d: %v", i, j, j, err)
+			continue
+		}
+		// read back, with 1-1 bytes overlaid
+		n, err := file.ReadAt(buf[:j+2], int64(j-1))
+		if err != nil {
+			t.Errorf("%d. error reading file: %v", i, err)
+			continue
+		}
+		buf = buf[:n]
+		if buf[0] == 0 {
+			t.Errorf("%d. file at %d has been overwritten with 0!", i, j-1)
+		}
+		if buf[n-1] == 0 {
+			t.Errorf("%d. file at %d has been overwritten with 0!", i, j-1+n)
+		}
+		for k, v := range buf[1 : n-1] {
+			if v != 0 {
+				t.Errorf("%d. error reading file at %d got %d, want 0.", i, k, v)
+			}
+		}
+	}
+}

Godeps/_workspace/src/github.com/cznic/fileutil/storage/LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of CZ.NIC nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

Godeps/_workspace/src/github.com/cznic/fileutil/storage/README

@@ -0,0 +1,5 @@
+This is a goinstall-able mirror of modified code already published at:
+https://git.nic.cz/redmine/projects/gofileutil/repository/show/storage
+
+Install: $go get github.com/cznic/fileutil/storage
+Godocs: http://gopkgdoc.appspot.com/pkg/github.com/cznic/fileutil/storage

Godeps/_workspace/src/github.com/cznic/fileutil/storage/all_test.go

@@ -0,0 +1,22 @@
+// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// blame: jnml, labs.nic.cz
+
+package storage
+
+import (
+	"flag"
+	"runtime"
+)
+
+var (
+	devFlag = flag.Bool("dev", false, "enable dev tests")
+	goFlag  = flag.Int("go", 1, "GOMAXPROCS")
+)
+
+func init() {
+	flag.Parse()
+	runtime.GOMAXPROCS(*goFlag)
+}

Godeps/_workspace/src/github.com/cznic/fileutil/storage/cache.go

@@ -0,0 +1,322 @@
+// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// blame: jnml, labs.nic.cz
+
+package storage
+
+import (
+	"container/list"
+	"io"
+	"math"
+	"os"
+	"sync"
+	"sync/atomic"
+)
+
+type cachepage struct {
+	b     [512]byte
+	dirty bool
+	lru   *list.Element
+	pi    int64
+	valid int // page content is b[:valid]
+}
+
+func (p *cachepage) wr(b []byte, off int) (wasDirty bool) {
+	copy(p.b[off:], b)
+	if n := off + len(b); n > p.valid {
+		p.valid = n
+	}
+	wasDirty = p.dirty
+	p.dirty = true
+	return
+}
+
+func (c *Cache) rd(off int64, read bool) (p *cachepage, ok bool) {
+	c.Rq++
+	pi := off >> 9
+	if p, ok = c.m[pi]; ok {
+		c.lru.MoveToBack(p.lru)
+		return
+	}
+
+	if !read {
+		return
+	}
+
+	fp := off &^ 511
+	if fp >= c.size {
+		return
+	}
+
+	rq := 512
+	if fp+512 > c.size {
+		rq = int(c.size - fp)
+	}
+	p = &cachepage{pi: pi, valid: rq}
+	p.lru = c.lru.PushBack(p)
+	if n, err := c.f.ReadAt(p.b[:p.valid], fp); n != rq {
+		panic(err)
+	}
+
+	c.Load++
+	if c.advise != nil {
+		c.advise(fp, 512, false)
+	}
+	c.m[pi], ok = p, true
+	return
+}
+
+func (c *Cache) wr(off int64) (p *cachepage) {
+	var ok bool
+	if p, ok = c.rd(off, false); ok {
+		return
+	}
+
+	pi := off >> 9
+	p = &cachepage{pi: pi}
+	p.lru = c.lru.PushBack(p)
+	c.m[pi] = p
+	return
+}
+
+// Cache provides caching support for another store Accessor.
+type Cache struct {
+	advise   func(int64, int, bool)
+	clean    chan bool
+	cleaning int32
+	close    chan bool
+	f        Accessor
+	fi       *FileInfo
+	lock     sync.Mutex
+	lru      *list.List
+	m        map[int64]*cachepage
+	maxpages int
+	size     int64
+	sync     chan bool
+	wlist    *list.List
+	write    chan bool
+	writing  int32
+	Rq       int64 // Pages requested from cache
+	Load     int64 // Pages loaded (cache miss)
+	Purge    int64 // Pages purged
+	Top      int   // "High water" pages
+}
+
+// Implementation of Accessor.
+func (c *Cache) BeginUpdate() error { return nil }
+
+// Implementation of Accessor.
+func (c *Cache) EndUpdate() error { return nil }
+
+// NewCache creates a caching Accessor from store with total of maxcache bytes.
+// NewCache returns the new Cache, implementing Accessor or an error if any.
+//
+// The LRU mechanism is used, so the cache tries to keep often accessed pages cached.
+//
+func NewCache(store Accessor, maxcache int64, advise func(int64, int, bool)) (c *Cache, err error) {
+	var fi os.FileInfo
+	if fi, err = store.Stat(); err != nil {
+		return
+	}
+
+	x := maxcache >> 9
+	if x > math.MaxInt32/2 {
+		x = math.MaxInt32 / 2
+	}
+	c = &Cache{
+		advise:   advise,
+		clean:    make(chan bool, 1),
+		close:    make(chan bool),
+		f:        store,
+		lru:      list.New(), // front == oldest used, back == last recently used
+		m:        make(map[int64]*cachepage),
+		maxpages: int(x),
+		size:     fi.Size(),
+		sync:     make(chan bool),
+		wlist:    list.New(),
+		write:    make(chan bool, 1),
+	}
+	c.fi = NewFileInfo(fi, c)
+	go c.writer()
+	go c.cleaner(int((int64(c.maxpages) * 95) / 100)) // hysteresis
+	return
+}
+
+func (c *Cache) Accessor() Accessor {
+	return c.f
+}
+
+func (c *Cache) Close() (err error) {
+	close(c.write)
+	<-c.close
+	close(c.clean)
+	<-c.close
+	return c.f.Close()
+}
+
+func (c *Cache) Name() (s string) {
+	return c.f.Name()
+}
+
+func (c *Cache) ReadAt(b []byte, off int64) (n int, err error) {
+	po := int(off) & 0x1ff
+	bp := 0
+	rem := len(b)
+	m := 0
+	for rem != 0 {
+		c.lock.Lock() // X1+
+		p, ok := c.rd(off, true)
+		if !ok {
+			c.lock.Unlock() // X1-
+			return -1, io.EOF
+		}
+
+		rq := rem
+		if po+rq > 512 {
+			rq = 512 - po
+		}
+		if n := copy(b[bp:bp+rq], p.b[po:p.valid]); n != rq {
+			c.lock.Unlock() // X1-
+			return -1, io.EOF
+		}
+
+		m = len(c.m)
+		c.lock.Unlock() // X1-
+		po = 0
+		bp += rq
+		off += int64(rq)
+		rem -= rq
+		n += rq
+	}
+	if m > c.maxpages && atomic.CompareAndSwapInt32(&c.cleaning, 0, 1) {
+		if m > c.Top {
+			c.Top = m
+		}
+		c.clean <- true
+	}
+	return
+}
+
+func (c *Cache) Stat() (fi os.FileInfo, err error) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	return c.fi, nil
+}
+
+func (c *Cache) Sync() (err error) {
+	c.write <- false
+	<-c.sync
+	return
+}
+
+func (c *Cache) Truncate(size int64) (err error) {
+	c.Sync() //TODO improve (discard pages, the writer goroutine should also be aware, ...)
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	c.size = size
+	return c.f.Truncate(size)
+}
+
+func (c *Cache) WriteAt(b []byte, off int64) (n int, err error) {
+	po := int(off) & 0x1ff
+	bp := 0
+	rem := len(b)
+	m := 0
+	for rem != 0 {
+		c.lock.Lock() // X+
+		p := c.wr(off)
+		rq := rem
+		if po+rq > 512 {
+			rq = 512 - po
+		}
+		if wasDirty := p.wr(b[bp:bp+rq], po); !wasDirty {
+			c.wlist.PushBack(p)
+		}
+		m = len(c.m)
+		po = 0
+		bp += rq
+		off += int64(rq)
+		if off > c.size {
+			c.size = off
+		}
+		c.lock.Unlock() // X-
+		rem -= rq
+		n += rq
+	}
+	if atomic.CompareAndSwapInt32(&c.writing, 0, 1) {
+		c.write <- true
+	}
+	if m > c.maxpages && atomic.CompareAndSwapInt32(&c.cleaning, 0, 1) {
+		if m > c.Top {
+			c.Top = m
+		}
+		c.clean <- true
+	}
+	return
+}
+
+func (c *Cache) writer() {
+	for ok := true; ok; {
+		var wr bool
+		var off int64
+		wr, ok = <-c.write
+		for {
+			c.lock.Lock() // X1+
+			item := c.wlist.Front()
+			if item == nil {
+				c.lock.Unlock() // X1-
+				break
+			}
+
+			p := item.Value.(*cachepage)
+			off = p.pi << 9
+			if n, err := c.f.WriteAt(p.b[:p.valid], off); n != p.valid {
+				c.lock.Unlock()                    // X1-
+				panic("TODO Cache.writer errchan") //TODO +errchan
+				panic(err)
+			}
+
+			p.dirty = false
+			c.wlist.Remove(item)
+			if c.advise != nil {
+				c.advise(off, 512, true)
+			}
+			c.lock.Unlock() // X1-
+		}
+		switch {
+		case wr:
+			atomic.AddInt32(&c.writing, -1)
+		case ok:
+			c.sync <- true
+		}
+	}
+	c.close <- true
+}
+
+func (c *Cache) cleaner(limit int) {
+	for _ = range c.clean {
+		var item *list.Element
+		for {
+			c.lock.Lock() // X1+
+			if len(c.m) < limit {
+				c.lock.Unlock() // X1-
+				break
+			}
+
+			if item == nil {
+				item = c.lru.Front()
+			}
+			if p := item.Value.(*cachepage); !p.dirty {
+				delete(c.m, p.pi)
+				c.lru.Remove(item)
+				c.Purge++
+			}
+			item = item.Next()
+			c.lock.Unlock() // X1-
+		}
+		atomic.AddInt32(&c.cleaning, -1)
+	}
+	c.close <- true
+}

Godeps/_workspace/src/github.com/cznic/fileutil/storage/cache_test.go

@@ -0,0 +1,83 @@
+// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// blame: jnml, labs.nic.cz
+
+package storage
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func newfile(t *testing.T) (string, string, Accessor) {
+	dir, err := ioutil.TempDir("", "test-storage-")
+	if err != nil {
+		panic(err)
+	}
+
+	name := filepath.Join(dir, "test.tmp")
+	f, err := NewFile(name, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0666)
+	if err != nil {
+		t.Fatal("newfile", err)
+	}
+
+	return dir, name, f
+}
+
+func readfile(t *testing.T, name string) (b []byte) {
+	var err error
+	if b, err = ioutil.ReadFile(name); err != nil {
+		t.Fatal("readfile")
+	}
+
+	return
+}
+
+func newcache(t *testing.T) (dir, name string, c *Cache) {
+	dir, name, f := newfile(t)
+	var err error
+	if c, err = NewCache(f, 1<<20, nil); err != nil {
+		t.Fatal("newCache", err)
+	}
+
+	return
+}
+
+func TestCache0(t *testing.T) {
+	dir, name, c := newcache(t)
+	defer os.RemoveAll(dir)
+
+	if err := c.Close(); err != nil {
+		t.Fatal(10, err)
+	}
+
+	if b := readfile(t, name); len(b) != 0 {
+		t.Fatal(20, len(b), 0)
+	}
+}
+
+func TestCache1(t *testing.T) {
+	dir, name, c := newcache(t)
+	defer os.RemoveAll(dir)
+
+	if n, err := c.WriteAt([]byte{0xa5}, 0); n != 1 {
+		t.Fatal(20, n, err)
+	}
+
+	if err := c.Close(); err != nil {
+		t.Fatal(10, err)
+	}
+
+	b := readfile(t, name)
+	if len(b) != 1 {
+		t.Fatal(30, len(b), 1)
+	}
+
+	if b[0] != 0xa5 {
+		t.Fatal(40, b[0], 0xa5)
+	}
+}

Godeps/_workspace/src/github.com/cznic/fileutil/storage/dev_test.go

@@ -0,0 +1,18 @@
+// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// blame: jnml, labs.nic.cz
+
+package storage
+
+import (
+	"testing"
+)
+
+func TestDevNothing(t *testing.T) {
+	if !*devFlag {
+		t.Log("not enabled")
+		return
+	}
+}

Godeps/_workspace/src/github.com/cznic/fileutil/storage/file.go

@@ -0,0 +1,50 @@
+// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// blame: jnml, labs.nic.cz
+
+package storage
+
+import (
+	"os"
+)
+
+// FileAccessor is the concrete type returned by NewFile and OpenFile.
+type FileAccessor struct {
+	*os.File
+}
+
+// Implementation of Accessor.
+func (f *FileAccessor) BeginUpdate() error { return nil }
+
+// Implementation of Accessor.
+func (f *FileAccessor) EndUpdate() error { return nil }
+
+// NewFile returns an Accessor backed by an os.File named name, It opens the
+// named file with specified flag (os.O_RDWR etc.) and perm, (0666 etc.) if
+// applicable.  If successful, methods on the returned Accessor can be used for
+// I/O.  It returns the Accessor and an Error, if any.
+//
+// NOTE: The returned Accessor implements BeginUpdate and EndUpdate as a no op.
+func NewFile(name string, flag int, perm os.FileMode) (store Accessor, err error) {
+	var f FileAccessor
+	if f.File, err = os.OpenFile(name, flag, perm); err == nil {
+		store = &f
+	}
+	return
+}
+
+// OpenFile returns an Accessor backed by an existing os.File named name, It
+// opens the named file with specified flag (os.O_RDWR etc.) and perm, (0666
+// etc.) if applicable.  If successful, methods on the returned Accessor can be
+// used for I/O.  It returns the Accessor and an Error, if any.
+//
+// NOTE: The returned Accessor implements BeginUpdate and EndUpdate as a no op.
+func OpenFile(name string, flag int, perm os.FileMode) (store Accessor, err error) {
+	var f FileAccessor
+	if f.File, err = os.OpenFile(name, flag, perm); err == nil {
+		store = &f
+	}
+	return
+}

Godeps/_workspace/src/github.com/cznic/fileutil/storage/mem.go

@@ -0,0 +1,161 @@
+// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// blame: jnml, labs.nic.cz
+
+package storage
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"math"
+	"os"
+)
+
+//TODO -> exported type w/ exported fields
+type memaccessor struct {
+	f  *os.File
+	fi *FileInfo
+	b  []byte
+}
+
+// Implementation of Accessor.
+func (m *memaccessor) BeginUpdate() error { return nil }
+
+// Implementation of Accessor.
+func (f *memaccessor) EndUpdate() error { return nil }
+
+// NewMem returns a new Accessor backed by an os.File.  The returned Accessor
+// keeps all of the store content in memory.  The memory and file images are
+// synced only by Sync and Close.  Recomended for small amounts of data only
+// and content which may be lost on process kill/crash.  NewMem return the
+// Accessor or an error of any.
+//
+// NOTE: The returned Accessor implements BeginUpdate and EndUpdate as a no op.
+func NewMem(f *os.File) (store Accessor, err error) {
+	a := &memaccessor{f: f}
+	if err = f.Truncate(0); err != nil {
+		return
+	}
+
+	var fi os.FileInfo
+	if fi, err = a.f.Stat(); err != nil {
+		return
+	}
+
+	a.fi = NewFileInfo(fi, a)
+	store = a
+	return
+}
+
+// OpenMem return a new Accessor backed by an os.File.  The store content is
+// loaded from f.  The returned Accessor keeps all of the store content in
+// memory.  The memory and file images are synced only Sync and Close.
+// Recomended for small amounts of data only and content which may be lost on
+// process kill/crash.  OpenMem return the Accessor or an error of any.
+//
+// NOTE: The returned Accessor implements BeginUpdate and EndUpdate as a no op.
+func OpenMem(f *os.File) (store Accessor, err error) {
+	a := &memaccessor{f: f}
+	if a.b, err = ioutil.ReadAll(a.f); err != nil {
+		a.f.Close()
+		return
+	}
+
+	var fi os.FileInfo
+	if fi, err = a.f.Stat(); err != nil {
+		a.f.Close()
+		return
+	}
+
+	a.fi = NewFileInfo(fi, a)
+	store = a
+	return
+}
+
+// Close implements Accessor. Specifically it synchronizes the memory and file images.
+func (a *memaccessor) Close() (err error) {
+	defer func() {
+		a.b = nil
+		if a.f != nil {
+			if e := a.f.Close(); e != nil && err == nil {
+				err = e
+			}
+		}
+		a.f = nil
+	}()
+
+	return a.Sync()
+}
+
+func (a *memaccessor) Name() string {
+	return a.f.Name()
+}
+
+func (a *memaccessor) ReadAt(b []byte, off int64) (n int, err error) {
+	if off < 0 || off > math.MaxInt32 {
+		return -1, fmt.Errorf("ReadAt: illegal offset %#x", off)
+	}
+
+	rq, fp := len(b), int(off)
+	if fp+rq > len(a.b) {
+		return -1, fmt.Errorf("ReadAt: illegal rq %#x @ offset %#x, len %#x", rq, fp, len(a.b))
+	}
+
+	copy(b, a.b[fp:])
+	return
+}
+
+func (a *memaccessor) Stat() (fi os.FileInfo, err error) {
+	i := a.fi
+	i.FSize = int64(len(a.b))
+	fi = i
+	return
+}
+
+// Sync implements Accessor. Specifically it synchronizes the memory and file images.
+func (a *memaccessor) Sync() (err error) {
+	var n int
+	if n, err = a.f.WriteAt(a.b, 0); n != len(a.b) {
+		return
+	}
+
+	return a.f.Truncate(int64(len(a.b)))
+}
+
+func (a *memaccessor) Truncate(size int64) (err error) {
+	defer func() {
+		if e := recover(); e != nil {
+			err = e.(error)
+		}
+	}()
+
+	if size > math.MaxInt32 {
+		panic(errors.New("truncate: illegal size"))
+	}
+
+	a.b = a.b[:int(size)]
+	return
+}
+
+func (a *memaccessor) WriteAt(b []byte, off int64) (n int, err error) {
+	if off < 0 || off > math.MaxInt32 {
+		return -1, errors.New("WriteAt: illegal offset")
+	}
+
+	rq, fp, size := len(b), int(off), len(a.b)
+	if need := rq + fp; need > size {
+		if need <= cap(a.b) {
+			a.b = a.b[:need]
+		} else {
+			nb := make([]byte, need, 2*need)
+			copy(nb, a.b)
+			a.b = nb
+		}
+	}
+
+	copy(a.b[int(off):], b)
+	return
+}

Godeps/_workspace/src/github.com/cznic/fileutil/storage/mem_test.go

@@ -0,0 +1,15 @@
+// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// blame: jnml, labs.nic.cz
+
+package storage
+
+import (
+	"testing"
+)
+
+func Test(t *testing.T) {
+	t.Log("TODO placeholder") //TODO
+}

Godeps/_workspace/src/github.com/cznic/fileutil/storage/probe.go

@@ -0,0 +1,74 @@
+// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// blame: jnml, labs.nic.cz
+
+package storage
+
+import "sync/atomic"
+
+// Probe collects usage statistics of the embeded Accessor.
+// Probe itself IS an Accessor.
+type Probe struct {
+	Accessor
+	Chain     *Probe
+	OpsRd     int64
+	OpsWr     int64
+	BytesRd   int64
+	BytesWr   int64
+	SectorsRd int64 // Assuming 512 byte sector size
+	SectorsWr int64
+}
+
+// NewProbe returns a newly created probe which embedes the src Accessor.
+// The retuned *Probe satisfies Accessor. if chain != nil then Reset()
+// is cascaded down the chained Probes.
+func NewProbe(src Accessor, chain *Probe) *Probe {
+	return &Probe{Accessor: src, Chain: chain}
+}
+
+func reset(n *int64) {
+	atomic.AddInt64(n, -atomic.AddInt64(n, 0))
+}
+
+// Reset zeroes the collected statistics of p.
+func (p *Probe) Reset() {
+	if p.Chain != nil {
+		p.Chain.Reset()
+	}
+	reset(&p.OpsRd)
+	reset(&p.OpsWr)
+	reset(&p.BytesRd)
+	reset(&p.BytesWr)
+	reset(&p.SectorsRd)
+	reset(&p.SectorsWr)
+}
+
+func (p *Probe) ReadAt(b []byte, off int64) (n int, err error) {
+	n, err = p.Accessor.ReadAt(b, off)
+	atomic.AddInt64(&p.OpsRd, 1)
+	atomic.AddInt64(&p.BytesRd, int64(n))
+	if n <= 0 {
+		return
+	}
+
+	sectorFirst := off >> 9
+	sectorLast := (off + int64(n) - 1) >> 9
+	atomic.AddInt64(&p.SectorsRd, sectorLast-sectorFirst+1)
+	return
+}
+
+func (p *Probe) WriteAt(b []byte, off int64) (n int, err error) {
+	n, err = p.Accessor.WriteAt(b, off)
+	atomic.AddInt64(&p.OpsWr, 1)
+	atomic.AddInt64(&p.BytesWr, int64(n))
+	if n <= 0 {
+		return
+	}
+
+	sectorFirst := off >> 9
+	sectorLast := (off + int64(n) - 1) >> 9
+	atomic.AddInt64(&p.SectorsWr, sectorLast-sectorFirst+1)
+	return
+}

Godeps/_workspace/src/github.com/cznic/fileutil/storage/probe_test.go

@@ -0,0 +1,86 @@
+// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// blame: jnml, labs.nic.cz
+
+package storage
+
+import (
+	"os"
+	"testing"
+)
+
+func (p *Probe) assert(t *testing.T, msg int, opsRd, opsWr, bytesRd, bytesWr, sectorsRd, sectorsWr int64) {
+	if n := p.OpsRd; n != opsRd {
+		t.Fatal(msg, n, opsRd)
+	}
+
+	if n := p.OpsWr; n != opsWr {
+		t.Fatal(msg+1, n, opsWr)
+	}
+
+	if n := p.BytesRd; n != bytesRd {
+		t.Fatal(msg+2, n, bytesRd)
+	}
+
+	if n := p.BytesWr; n != bytesWr {
+		t.Fatal(msg+3, n, bytesWr)
+	}
+
+	if n := p.SectorsRd; n != sectorsRd {
+		t.Fatal(msg+4, n, sectorsRd)
+	}
+
+	if n := p.SectorsWr; n != sectorsWr {
+		t.Fatal(msg+5, n, sectorsWr)
+	}
+}
+
+func TestProbe(t *testing.T) {
+	return //TODO disabled due to atomic.AddInt64 failing on W32
+	const fn = "test.tmp"
+
+	store, err := NewFile(fn, os.O_CREATE|os.O_RDWR|os.O_CREATE, 0666)
+	if err != nil {
+		t.Fatal(10, err)
+	}
+
+	defer func() {
+		ec := store.Close()
+		er := os.Remove(fn)
+		if ec != nil {
+			t.Fatal(10000, ec)
+		}
+		if er != nil {
+			t.Fatal(10001, er)
+		}
+	}()
+
+	probe := NewProbe(store, nil)
+	if n, err := probe.WriteAt([]byte{1}, 0); n != 1 {
+		t.Fatal(20, err)
+	}
+
+	probe.assert(t, 30, 0, 1, 0, 1, 0, 1)
+	b := []byte{0}
+	if n, err := probe.ReadAt(b, 0); n != 1 {
+		t.Fatal(40, err)
+	}
+
+	if n := b[0]; n != 1 {
+		t.Fatal(50, n, 1)
+	}
+
+	probe.assert(t, 60, 1, 1, 1, 1, 1, 1)
+	if n, err := probe.WriteAt([]byte{2, 3}, 510); n != 2 {
+		t.Fatal(70, err)
+	}
+
+	probe.assert(t, 80, 1, 2, 1, 3, 1, 2)
+	if n, err := probe.WriteAt([]byte{2, 3}, 511); n != 2 {
+		t.Fatal(90, err)
+	}
+
+	probe.assert(t, 100, 1, 3, 1, 5, 1, 4)
+}

Godeps/_workspace/src/github.com/cznic/fileutil/storage/storage.go

@@ -0,0 +1,141 @@
+// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// blame: jnml, labs.nic.cz
+
+// WIP: Package storage defines and implements storage providers and store accessors.
+package storage
+
+import (
+	"os"
+	"sync"
+	"time"
+)
+
+// FileInfo is a type implementing os.FileInfo which has setable fields, like
+// the older os.FileInfo used to have. It is used wehere e.g. the Size is
+// needed to be faked (encapsulated/memory only file, file cache, etc.).
+type FileInfo struct {
+	FName    string      // base name of the file
+	FSize    int64       // length in bytes
+	FMode    os.FileMode // file mode bits
+	FModTime time.Time   // modification time
+	FIsDir   bool        // abbreviation for Mode().IsDir()
+	sys      interface{} // underlying data source (can be nil)
+}
+
+// NewFileInfo creates FileInfo from os.FileInfo fi.
+func NewFileInfo(fi os.FileInfo, sys interface{}) *FileInfo {
+	return &FileInfo{fi.Name(), fi.Size(), fi.Mode(), fi.ModTime(), fi.IsDir(), sys}
+}
+
+// Implementation of os.FileInfo
+func (fi *FileInfo) Name() string {
+	return fi.FName
+}
+
+// Implementation of os.FileInfo
+func (fi *FileInfo) Size() int64 {
+	return fi.FSize
+}
+
+// Implementation of os.FileInfo
+func (fi *FileInfo) Mode() os.FileMode {
+	return fi.FMode
+}
+
+// Implementation of os.FileInfo
+func (fi *FileInfo) ModTime() time.Time {
+	return fi.FModTime
+}
+
+// Implementation of os.FileInfo
+func (fi *FileInfo) IsDir() bool {
+	return fi.FIsDir
+}
+
+func (fi *FileInfo) Sys() interface{} {
+	return fi.sys
+}
+
+// Accessor provides I/O methods to access a store.
+type Accessor interface {
+
+	// Close closes the store, rendering it unusable for I/O. It returns an
+	// error, if any.
+	Close() error
+
+	// Name returns the name of the file as presented to Open.
+	Name() string
+
+	// ReadAt reads len(b) bytes from the store starting at byte offset off.
+	// It returns the number of bytes read and the error, if any.
+	// EOF is signaled by a zero count with err set to os.EOF.
+	// ReadAt always returns a non-nil Error when n != len(b).
+	ReadAt(b []byte, off int64) (n int, err error)
+
+	// Stat returns the FileInfo structure describing the store. It returns
+	// the os.FileInfo and an error, if any.
+	Stat() (fi os.FileInfo, err error)
+
+	// Sync commits the current contents of the store to stable storage.
+	// Typically, this means flushing the file system's in-memory copy of
+	// recently written data to disk.
+	Sync() (err error)
+
+	// Truncate changes the size of the store. It does not change the I/O
+	// offset.
+	Truncate(size int64) error
+
+	// WriteAt writes len(b) bytes to the store starting at byte offset off.
+	// It returns the number of bytes written and an error, if any.
+	// WriteAt returns a non-nil Error when n != len(b).
+	WriteAt(b []byte, off int64) (n int, err error)
+
+	// Before every [structural] change of a store the BeginUpdate is to be
+	// called and paired with EndUpdate after the change makes the store's
+	// state consistent again. Invocations of BeginUpdate may nest. On
+	// invoking the last non nested EndUpdate an implicit "commit" should
+	// be performed by the store/provider. The concrete mechanism is
+	// unspecified. It could be for example a write-ahead log.  Stores may
+	// implement BeginUpdate and EndUpdate as a (documented) no op.
+	BeginUpdate() error
+	EndUpdate() error
+}
+
+// Mutate is a helper/wrapper for executing f in between a.BeginUpdate and
+// a.EndUpdate.  Any parameters and/or return values except an error should be
+// captured by a function literal passed as f. The returned err is either nil
+// or the first non nil error returned from the sequence of execution:
+// BeginUpdate, [f,] EndUpdate. The pair BeginUpdate/EndUpdate *is* invoked
+// always regardles of any possible errors produced.  Mutate doesn't handle
+// panic, it should be used only with a function [literal] which doesn't panic.
+// Otherwise the pairing of BeginUpdate/EndUpdate is not guaranteed.
+//
+// NOTE: If BeginUpdate, which is invoked before f, returns a non-nil error,
+// then f is not invoked at all (but EndUpdate still is).
+func Mutate(a Accessor, f func() error) (err error) {
+	defer func() {
+		if e := a.EndUpdate(); e != nil && err == nil {
+			err = e
+		}
+	}()
+
+	if err = a.BeginUpdate(); err != nil {
+		return
+	}
+
+	return f()
+}
+
+// LockedMutate wraps Mutate in yet another layer consisting of a
+// l.Lock/l.Unlock pair. All other limitations apply as in Mutate, e.g. no
+// panics are allowed to happen - otherwise no guarantees can be made about
+// Unlock matching the Lock.
+func LockedMutate(a Accessor, l sync.Locker, f func() error) (err error) {
+	l.Lock()
+	defer l.Unlock()
+
+	return Mutate(a, f)
+}

Godeps/_workspace/src/github.com/cznic/fileutil/storage/test_deps.go

@@ -0,0 +1,13 @@
+// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// blame: jnml, labs.nic.cz
+
+package storage
+
+// Pull test dependencies too.
+// Enables easy 'go test X' after 'go get X'
+import (
+// nothing yet
+)

Godeps/_workspace/src/github.com/cznic/fileutil/test_deps.go

@@ -0,0 +1,13 @@
+// Copyright (c) 2014 The fileutil Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// blame: jnml, labs.nic.cz
+
+package fileutil
+
+// Pull test dependencies too.
+// Enables easy 'go test X' after 'go get X'
+import (
+// nothing yet
+)

Godeps/_workspace/src/github.com/cznic/mathutil/AUTHORS

@@ -0,0 +1,12 @@
+# This file lists authors for copyright purposes.  This file is distinct from
+# the CONTRIBUTORS files.  See the latter for an explanation.
+#
+# Names should be added to this file as:
+#     Name or Organization <email address>
+#
+# The email address is not required for organizations.
+#
+# Please keep the list sorted.
+
+CZ.NIC z.s.p.o. <kontakt@nic.cz>
+Jan Mercl <0xjnml@gmail.com>

Godeps/_workspace/src/github.com/cznic/mathutil/CONTRIBUTORS

@@ -0,0 +1,10 @@
+# This file lists people who contributed code to this repository.  The AUTHORS
+# file lists the copyright holders; this file lists people.
+#
+# Names should be added to this file like so:
+#     Name <email address>
+#
+# Please keep the list sorted.
+
+Gary Burd <gary@beagledreams.com>
+Jan Mercl <0xjnml@gmail.com>

Godeps/_workspace/src/github.com/cznic/mathutil/LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2014 The mathutil Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the names of the authors nor the names of the
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

Godeps/_workspace/src/github.com/cznic/mathutil/Makefile

@@ -0,0 +1,31 @@
+# Copyright (c) 2014 The mathutil Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+.PHONY: all todo clean nuke
+
+grep=--include=*.go --include=*.run --include=*.y
+
+all: editor
+	go build
+	go vet || true
+	golint .
+	go install
+	make todo
+
+clean:
+	go clean
+
+editor:
+	go fmt
+	go test -i
+	go test
+
+todo:
+	@grep -nr $(grep) ^[[:space:]]*_[[:space:]]*=[[:space:]][[:alpha:]][[:alnum:]]* * || true
+	@grep -nr $(grep) TODO * || true
+	@grep -nr $(grep) BUG * || true
+	@grep -nr $(grep) println * || true
+
+nuke: clean
+	go clean -i

Godeps/_workspace/src/github.com/cznic/mathutil/README

@@ -0,0 +1,10 @@
+This is a goinstall-able mirror of modified code already published at:
+http://git.nic.cz/redmine/projects/gornd/repository
+
+Packages in this repository:
+
+Install: $ go get github.com/cznic/mathutil
+Godocs:  http://godoc.org/github.com/cznic/mathutil
+
+Install: $ go get github.com/cznic/mathutil/mersenne
+Godocs:  http://godoc.org/github.com/cznic/mathutil/mersenne

Godeps/_workspace/src/github.com/cznic/mathutil/bits.go

@@ -0,0 +1,207 @@
+// Copyright (c) 2014 The mathutil Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package mathutil
+
+import (
+	"math/big"
+)
+
+// BitLenByte returns the bit width of the non zero part of n.
+func BitLenByte(n byte) int {
+	return log2[n] + 1
+}
+
+// BitLenUint16 returns the bit width of the non zero part of n.
+func BitLenUint16(n uint16) int {
+	if b := n >> 8; b != 0 {
+		return log2[b] + 8 + 1
+	}
+
+	return log2[n] + 1
+}
+
+// BitLenUint32 returns the bit width of the non zero part of n.
+func BitLenUint32(n uint32) int {
+	if b := n >> 24; b != 0 {
+		return log2[b] + 24 + 1
+	}
+
+	if b := n >> 16; b != 0 {
+		return log2[b] + 16 + 1
+	}
+
+	if b := n >> 8; b != 0 {
+		return log2[b] + 8 + 1
+	}
+
+	return log2[n] + 1
+}
+
+// BitLen returns the bit width of the non zero part of n.
+func BitLen(n int) int { // Should handle correctly [future] 64 bit Go ints
+	if IntBits == 64 {
+		return BitLenUint64(uint64(n))
+	}
+
+	if b := byte(n >> 24); b != 0 {
+		return log2[b] + 24 + 1
+	}
+
+	if b := byte(n >> 16); b != 0 {
+		return log2[b] + 16 + 1
+	}
+
+	if b := byte(n >> 8); b != 0 {
+		return log2[b] + 8 + 1
+	}
+
+	return log2[byte(n)] + 1
+}
+
+// BitLenUint returns the bit width of the non zero part of n.
+func BitLenUint(n uint) int { // Should handle correctly [future] 64 bit Go uints
+	if IntBits == 64 {
+		return BitLenUint64(uint64(n))
+	}
+
+	if b := n >> 24; b != 0 {
+		return log2[b] + 24 + 1
+	}
+
+	if b := n >> 16; b != 0 {
+		return log2[b] + 16 + 1
+	}
+
+	if b := n >> 8; b != 0 {
+		return log2[b] + 8 + 1
+	}
+
+	return log2[n] + 1
+}
+
+// BitLenUint64 returns the bit width of the non zero part of n.
+func BitLenUint64(n uint64) int {
+	if b := n >> 56; b != 0 {
+		return log2[b] + 56 + 1
+	}
+
+	if b := n >> 48; b != 0 {
+		return log2[b] + 48 + 1
+	}
+
+	if b := n >> 40; b != 0 {
+		return log2[b] + 40 + 1
+	}
+
+	if b := n >> 32; b != 0 {
+		return log2[b] + 32 + 1
+	}
+
+	if b := n >> 24; b != 0 {
+		return log2[b] + 24 + 1
+	}
+
+	if b := n >> 16; b != 0 {
+		return log2[b] + 16 + 1
+	}
+
+	if b := n >> 8; b != 0 {
+		return log2[b] + 8 + 1
+	}
+
+	return log2[n] + 1
+}
+
+// BitLenUintptr returns the bit width of the non zero part of n.
+func BitLenUintptr(n uintptr) int {
+	if b := n >> 56; b != 0 {
+		return log2[b] + 56 + 1
+	}
+
+	if b := n >> 48; b != 0 {
+		return log2[b] + 48 + 1
+	}
+
+	if b := n >> 40; b != 0 {
+		return log2[b] + 40 + 1
+	}
+
+	if b := n >> 32; b != 0 {
+		return log2[b] + 32 + 1
+	}
+
+	if b := n >> 24; b != 0 {
+		return log2[b] + 24 + 1
+	}
+
+	if b := n >> 16; b != 0 {
+		return log2[b] + 16 + 1
+	}
+
+	if b := n >> 8; b != 0 {
+		return log2[b] + 8 + 1
+	}
+
+	return log2[n] + 1
+}
+
+// PopCountByte returns population count of n (number of bits set in n).
+func PopCountByte(n byte) int {
+	return int(popcnt[byte(n)])
+}
+
+// PopCountUint16 returns population count of n (number of bits set in n).
+func PopCountUint16(n uint16) int {
+	return int(popcnt[byte(n>>8)]) + int(popcnt[byte(n)])
+}
+
+// PopCountUint32 returns population count of n (number of bits set in n).
+func PopCountUint32(n uint32) int {
+	return int(popcnt[byte(n>>24)]) + int(popcnt[byte(n>>16)]) +
+		int(popcnt[byte(n>>8)]) + int(popcnt[byte(n)])
+}
+
+// PopCount returns population count of n (number of bits set in n).
+func PopCount(n int) int { // Should handle correctly [future] 64 bit Go ints
+	if IntBits == 64 {
+		return PopCountUint64(uint64(n))
+	}
+
+	return PopCountUint32(uint32(n))
+}
+
+// PopCountUint returns population count of n (number of bits set in n).
+func PopCountUint(n uint) int { // Should handle correctly [future] 64 bit Go uints
+	if IntBits == 64 {
+		return PopCountUint64(uint64(n))
+	}
+
+	return PopCountUint32(uint32(n))
+}
+
+// PopCountUintptr returns population count of n (number of bits set in n).
+func PopCountUintptr(n uintptr) int {
+	if UintPtrBits == 64 {
+		return PopCountUint64(uint64(n))
+	}
+
+	return PopCountUint32(uint32(n))
+}
+
+// PopCountUint64 returns population count of n (number of bits set in n).
+func PopCountUint64(n uint64) int {
+	return int(popcnt[byte(n>>56)]) + int(popcnt[byte(n>>48)]) +
+		int(popcnt[byte(n>>40)]) + int(popcnt[byte(n>>32)]) +
+		int(popcnt[byte(n>>24)]) + int(popcnt[byte(n>>16)]) +
+		int(popcnt[byte(n>>8)]) + int(popcnt[byte(n)])
+}
+
+// PopCountBigInt returns population count of |n| (number of bits set in |n|).
+func PopCountBigInt(n *big.Int) (r int) {
+	for _, v := range n.Bits() {
+		r += PopCountUintptr(uintptr(v))
+	}
+	return
+}

Godeps/_workspace/src/github.com/cznic/mathutil/envelope.go

@@ -0,0 +1,46 @@
+// Copyright (c) 2014 The mathutil Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package mathutil
+
+import (
+	"math"
+)
+
+// Approximation type determines approximation methods used by e.g. Envelope.
+type Approximation int
+
+// Specific approximation method tags
+const (
+	_          Approximation = iota
+	Linear                   // As named
+	Sinusoidal               // Smooth for all derivations
+)
+
+// Envelope is an utility for defining simple curves using a small (usually)
+// set of data points.  Envelope returns a value defined by x, points and
+// approximation.  The value of x must be in [0,1) otherwise the result is
+// undefined or the function may panic. Points are interpreted as dividing the
+// [0,1) interval in len(points)-1 sections, so len(points) must be > 1 or the
+// function may panic. According to the left and right points closing/adjacent
+// to the section the resulting value is interpolated using the chosen
+// approximation method.  Unsupported values of approximation are silently
+// interpreted as 'Linear'.
+func Envelope(x float64, points []float64, approximation Approximation) float64 {
+	step := 1 / float64(len(points)-1)
+	fslot := math.Floor(x / step)
+	mod := x - fslot*step
+	slot := int(fslot)
+	l, r := points[slot], points[slot+1]
+	rmod := mod / step
+	switch approximation {
+	case Sinusoidal:
+		k := (math.Sin(math.Pi*(rmod-0.5)) + 1) / 2
+		return l + (r-l)*k
+	case Linear:
+		fallthrough
+	default:
+		return l + (r-l)*rmod
+	}
+}

Godeps/_workspace/src/github.com/cznic/mathutil/example/.gitignore

@@ -0,0 +1,2 @@
+example
+Makefile

Godeps/_workspace/src/github.com/cznic/mathutil/example/example.go

@@ -0,0 +1,48 @@
+// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// blame: jnml, labs.nic.cz
+
+// +build ignore
+
+package main
+
+import (
+	"bufio"
+	"flag"
+	"github.com/coreos/rocket/Godeps/_workspace/src/github.com/cznic/mathutil"
+	"log"
+	"math"
+	"os"
+)
+
+/*
+
+$ # Usage e.g.:
+$ go run example.go -max 1024 > mathutil.dat # generate 1kB of "random" data
+
+*/
+func main() {
+	r, err := mathutil.NewFC32(math.MinInt32, math.MaxInt32, true)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	var mflag uint64
+	flag.Uint64Var(&mflag, "max", 0, "limit output to max bytes")
+	flag.Parse()
+	stdout := bufio.NewWriter(os.Stdout)
+	if mflag != 0 {
+		for i := uint64(0); i < mflag; i++ {
+			if err := stdout.WriteByte(byte(r.Next())); err != nil {
+				log.Fatal(err)
+			}
+		}
+		stdout.Flush()
+		return
+	}
+
+	for stdout.WriteByte(byte(r.Next())) == nil {
+	}
+}

Godeps/_workspace/src/github.com/cznic/mathutil/example2/.gitignore

@@ -0,0 +1,2 @@
+example2
+Makefile

Godeps/_workspace/src/github.com/cznic/mathutil/example2/example2.go

@@ -0,0 +1,66 @@
+// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// blame: jnml, labs.nic.cz
+
+// +build ignore
+
+package main
+
+import (
+	"bytes"
+	"github.com/coreos/rocket/Godeps/_workspace/src/github.com/cznic/mathutil"
+	"image"
+	"image/png"
+	"io/ioutil"
+	"log"
+	"math"
+	"math/rand"
+)
+
+// $ go run example2.go # view rand.png and rnd.png by your favorite pic viewer
+//
+// see http://www.boallen.com/random-numbers.html
+func main() {
+	sqr := image.Rect(0, 0, 511, 511)
+	r, err := mathutil.NewFC32(math.MinInt32, math.MaxInt32, true)
+	if err != nil {
+		log.Fatal("NewFC32", err)
+	}
+
+	img := image.NewGray(sqr)
+	for y := 0; y < 512; y++ {
+		for x := 0; x < 512; x++ {
+			if r.Next()&1 != 0 {
+				img.Set(x, y, image.White)
+			}
+		}
+	}
+	buf := bytes.NewBuffer(nil)
+	if err := png.Encode(buf, img); err != nil {
+		log.Fatal("Encode rnd.png ", err)
+	}
+
+	if err := ioutil.WriteFile("rnd.png", buf.Bytes(), 0666); err != nil {
+		log.Fatal("ioutil.WriteFile/rnd.png ", err)
+	}
+
+	r2 := rand.New(rand.NewSource(0))
+	img = image.NewGray(sqr)
+	for y := 0; y < 512; y++ {
+		for x := 0; x < 512; x++ {
+			if r2.Int()&1 != 0 {
+				img.Set(x, y, image.White)
+			}
+		}
+	}
+	buf = bytes.NewBuffer(nil)
+	if err := png.Encode(buf, img); err != nil {
+		log.Fatal("Encode rand.png ", err)
+	}
+
+	if err := ioutil.WriteFile("rand.png", buf.Bytes(), 0666); err != nil {
+		log.Fatal("ioutil.WriteFile/rand.png ", err)
+	}
+}