1	/*
     2	Copyright 2011 The Perkeep Authors
     3	
     4	Licensed under the Apache License, Version 2.0 (the "License");
     5	you may not use this file except in compliance with the License.
     6	You may obtain a copy of the License at
     7	
     8	     http://www.apache.org/licenses/LICENSE-2.0
     9	
    10	Unless required by applicable law or agreed to in writing, software
    11	distributed under the License is distributed on an "AS IS" BASIS,
    12	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13	See the License for the specific language governing permissions and
    14	limitations under the License.
    15	*/
    16	
    17	// Package indextest contains the unit tests for the indexer so they
    18	// can be re-used for each specific implementation of the index
    19	// Storage interface.
    20	package indextest // import "perkeep.org/pkg/index/indextest"
    21	
    22	import (
    23		"bytes"
    24		"context"
    25		"errors"
    26		"fmt"
    27		"log"
    28		"net/url"
    29		"os"
    30		"path/filepath"
    31		"reflect"
    32		"strings"
    33		"testing"
    34		"time"
    35	
    36		"perkeep.org/internal/osutil"
    37		"perkeep.org/pkg/blob"
    38		"perkeep.org/pkg/index"
    39		"perkeep.org/pkg/jsonsign"
    40		"perkeep.org/pkg/schema"
    41		"perkeep.org/pkg/sorted"
    42		"perkeep.org/pkg/test"
    43		"perkeep.org/pkg/types/camtypes"
    44	)
    45	
    46	var ctxbg = context.Background()
    47	
    48	// An IndexDeps is a helper for populating and querying an Index for tests.
    49	type IndexDeps struct {
    50		Index *index.Index
    51	
    52		BlobSource *test.Fetcher
    53	
    54		// Following three needed for signing:
    55		PublicKeyFetcher *test.Fetcher
    56		EntityFetcher    jsonsign.EntityFetcher // fetching decrypted openpgp entities
    57		SignerBlobRef    blob.Ref
    58	
    59		now time.Time // fake clock, nanos since epoch
    60	
    61		Fataler // optional means of failing.
    62	}
    63	
    64	type Fataler interface {
    65		Fatalf(format string, args ...interface{})
    66	}
    67	
    68	type logFataler struct{}
    69	
    70	func (logFataler) Fatalf(format string, args ...interface{}) {
    71		log.Fatalf(format, args...)
    72	}
    73	
    74	func (id *IndexDeps) Get(key string) string {
    75		v, _ := id.Index.Storage().Get(key)
    76		return v
    77	}
    78	
    79	func (id *IndexDeps) Set(key, value string) error {
    80		return id.Index.Storage().Set(key, value)
    81	}
    82	
    83	func (id *IndexDeps) DumpIndex(t *testing.T) {
    84		t.Logf("Begin index dump:")
    85		it := id.Index.Storage().Find("", "")
    86		for it.Next() {
    87			t.Logf("  %q = %q", it.Key(), it.Value())
    88		}
    89		if err := it.Close(); err != nil {
    90			t.Fatalf("iterator close = %v", err)
    91		}
    92		t.Logf("End index dump.")
    93	}
    94	
    95	func (id *IndexDeps) Sign(m *schema.Builder) *test.Blob {
    96		m.SetSigner(id.SignerBlobRef)
    97		unsigned, err := m.JSON()
    98		if err != nil {
    99			id.Fatalf("uploadAndSignMap: " + err.Error())
   100		}
   101		sr := &jsonsign.SignRequest{
   102			UnsignedJSON:  unsigned,
   103			Fetcher:       id.PublicKeyFetcher,
   104			EntityFetcher: id.EntityFetcher,
   105			SignatureTime: id.now,
   106		}
   107		signed, err := sr.Sign(ctxbg)
   108		if err != nil {
   109			id.Fatalf("problem signing: " + err.Error())
   110		}
   111		tb := &test.Blob{Contents: signed}
   112		return tb
   113	}
   114	
   115	func (id *IndexDeps) Upload(tb *test.Blob) blob.Ref {
   116		_, err := id.BlobSource.ReceiveBlob(ctxbg, tb.BlobRef(), tb.Reader())
   117		if err != nil {
   118			id.Fatalf("public uploading signed blob to blob source, pre-indexing: %v, %v", tb.BlobRef(), err)
   119		}
   120		_, err = id.Index.ReceiveBlob(ctxbg, tb.BlobRef(), tb.Reader())
   121		if err != nil {
   122			id.Fatalf("problem indexing blob: %v\nblob was:\n%s", err, tb.Contents)
   123		}
   124		return tb.BlobRef()
   125	}
   126	
   127	func (id *IndexDeps) uploadAndSign(m *schema.Builder) blob.Ref {
   128		return id.Upload(id.Sign(m))
   129	}
   130	
   131	// NewPermanode creates (& signs) a new permanode and adds it
   132	// to the index, returning its blobref.
   133	func (id *IndexDeps) NewPermanode() blob.Ref {
   134		unsigned := schema.NewUnsignedPermanode()
   135		return id.uploadAndSign(unsigned)
   136	}
   137	
   138	// NewPlannedPermanode creates (& signs) a new planned permanode and adds it
   139	// to the index, returning its blobref.
   140	func (id *IndexDeps) NewPlannedPermanode(key string) blob.Ref {
   141		unsigned := schema.NewPlannedPermanode(key)
   142		return id.uploadAndSign(unsigned)
   143	}
   144	
   145	func (id *IndexDeps) advanceTime() time.Time {
   146		id.now = id.now.Add(1 * time.Second)
   147		return id.now
   148	}
   149	
   150	// LastTime returns the time of the most recent mutation (claim).
   151	func (id *IndexDeps) LastTime() time.Time {
   152		return id.now
   153	}
   154	
   155	func (id *IndexDeps) SetAttribute(permaNode blob.Ref, attr, value string) blob.Ref {
   156		m := schema.NewSetAttributeClaim(permaNode, attr, value)
   157		m.SetClaimDate(id.advanceTime())
   158		return id.uploadAndSign(m)
   159	}
   160	
   161	func (id *IndexDeps) SetAttribute_NoTimeMove(permaNode blob.Ref, attr, value string) blob.Ref {
   162		m := schema.NewSetAttributeClaim(permaNode, attr, value)
   163		m.SetClaimDate(id.LastTime())
   164		return id.uploadAndSign(m)
   165	}
   166	
   167	func (id *IndexDeps) AddAttribute(permaNode blob.Ref, attr, value string) blob.Ref {
   168		m := schema.NewAddAttributeClaim(permaNode, attr, value)
   169		m.SetClaimDate(id.advanceTime())
   170		return id.uploadAndSign(m)
   171	}
   172	
   173	func (id *IndexDeps) DelAttribute(permaNode blob.Ref, attr, value string) blob.Ref {
   174		m := schema.NewDelAttributeClaim(permaNode, attr, value)
   175		m.SetClaimDate(id.advanceTime())
   176		return id.uploadAndSign(m)
   177	}
   178	
   179	func (id *IndexDeps) Delete(target blob.Ref) blob.Ref {
   180		m := schema.NewDeleteClaim(target)
   181		m.SetClaimDate(id.advanceTime())
   182		return id.uploadAndSign(m)
   183	}
   184	
   185	var noTime = time.Time{}
   186	
   187	func (id *IndexDeps) UploadString(v string) blob.Ref {
   188		cb := &test.Blob{Contents: v}
   189		id.BlobSource.AddBlob(cb)
   190		br := cb.BlobRef()
   191		_, err := id.Index.ReceiveBlob(ctxbg, br, cb.Reader())
   192		if err != nil {
   193			id.Fatalf("UploadString: %v", err)
   194		}
   195		return br
   196	}
   197	
   198	// If modTime is zero, it's not used.
   199	func (id *IndexDeps) UploadFile(fileName string, contents string, modTime time.Time) (fileRef, wholeRef blob.Ref) {
   200		wholeRef = id.UploadString(contents)
   201	
   202		m := schema.NewFileMap(fileName)
   203		m.PopulateParts(int64(len(contents)), []schema.BytesPart{
   204			{
   205				Size:    uint64(len(contents)),
   206				BlobRef: wholeRef,
   207			}})
   208		if !modTime.IsZero() {
   209			m.SetModTime(modTime)
   210		}
   211		fjson, err := m.JSON()
   212		if err != nil {
   213			id.Fatalf("UploadFile.JSON: %v", err)
   214		}
   215		fb := &test.Blob{Contents: fjson}
   216		id.BlobSource.AddBlob(fb)
   217		fileRef = fb.BlobRef()
   218		_, err = id.Index.ReceiveBlob(ctxbg, fileRef, fb.Reader())
   219		if err != nil {
   220			panic(err)
   221		}
   222		return
   223	}
   224	
   225	// If modTime is zero, it's not used.
   226	func (id *IndexDeps) UploadDir(dirName string, children []blob.Ref, modTime time.Time) blob.Ref {
   227		// static-set entries blob
   228		ss := schema.NewStaticSet()
   229		ss.SetStaticSetMembers(children)
   230		ssjson := ss.Blob().JSON()
   231		ssb := &test.Blob{Contents: ssjson}
   232		id.BlobSource.AddBlob(ssb)
   233		_, err := id.Index.ReceiveBlob(ctxbg, ssb.BlobRef(), ssb.Reader())
   234		if err != nil {
   235			id.Fatalf("UploadDir.ReceiveBlob: %v", err)
   236		}
   237	
   238		// directory blob
   239		bb := schema.NewDirMap(dirName)
   240		bb.PopulateDirectoryMap(ssb.BlobRef())
   241		if !modTime.IsZero() {
   242			bb.SetModTime(modTime)
   243		}
   244		dirjson, err := bb.JSON()
   245		if err != nil {
   246			id.Fatalf("UploadDir.JSON: %v", err)
   247		}
   248		dirb := &test.Blob{Contents: dirjson}
   249		id.BlobSource.AddBlob(dirb)
   250		_, err = id.Index.ReceiveBlob(ctxbg, dirb.BlobRef(), dirb.Reader())
   251		if err != nil {
   252			id.Fatalf("UploadDir.ReceiveBlob: %v", err)
   253		}
   254		return dirb.BlobRef()
   255	}
   256	
   257	var (
   258		PubKey = &test.Blob{Contents: `-----BEGIN PGP PUBLIC KEY BLOCK-----
   259	
   260	xsBNBEzgoVsBCAC/56aEJ9BNIGV9FVP+WzenTAkg12k86YqlwJVAB/VwdMlyXxvi
   261	bCT1RVRfnYxscs14LLfcMWF3zMucw16mLlJCBSLvbZ0jn4h+/8vK5WuAdjw2YzLs
   262	WtBcjWn3lV6tb4RJz5gtD/o1w8VWxwAnAVIWZntKAWmkcChCRgdUeWso76+plxE5
   263	aRYBJqdT1mctGqNEISd/WYPMgwnWXQsVi3x4z1dYu2tD9uO1dkAff12z1kyZQIBQ
   264	rexKYRRRh9IKAayD4kgS0wdlULjBU98aeEaMz1ckuB46DX3lAYqmmTEL/Rl9cOI0
   265	Enpn/oOOfYFa5h0AFndZd1blMvruXfdAobjVABEBAAE=
   266	=28/7
   267	-----END PGP PUBLIC KEY BLOCK-----`}
   268		KeyID = "2931A67C26F5ABDA"
   269	)
   270	
   271	// NewIndexDeps returns an IndexDeps helper for populating and working
   272	// with the provided index for tests.
   273	func NewIndexDeps(index *index.Index) *IndexDeps {
   274		camliRootPath, err := osutil.GoPackagePath("perkeep.org")
   275		if err != nil {
   276			log.Fatal("Package perkeep.org not found in $GOPATH or $GOPATH not defined")
   277		}
   278		secretRingFile := filepath.Join(camliRootPath, "pkg", "jsonsign", "testdata", "test-secring.gpg")
   279	
   280		id := &IndexDeps{
   281			Index:            index,
   282			BlobSource:       new(test.Fetcher),
   283			PublicKeyFetcher: new(test.Fetcher),
   284			EntityFetcher: &jsonsign.CachingEntityFetcher{
   285				Fetcher: &jsonsign.FileEntityFetcher{File: secretRingFile},
   286			},
   287			SignerBlobRef: PubKey.BlobRef(),
   288			now:           test.ClockOrigin,
   289			Fataler:       logFataler{},
   290		}
   291		// Add dev client test key public key, keyid 26F5ABDA,
   292		// blobref sha224-a794846212ff67acdd00c6b90eee492baf674d41da8a621d2e8042dd
   293		if g, w := id.SignerBlobRef.String(), "sha224-a794846212ff67acdd00c6b90eee492baf674d41da8a621d2e8042dd"; g != w {
   294			id.Fatalf("unexpected signer blobref; got signer = %q; want %q", g, w)
   295		}
   296		id.PublicKeyFetcher.AddBlob(PubKey)
   297		id.Index.KeyFetcher = id.PublicKeyFetcher
   298		id.Index.InitBlobSource(id.BlobSource)
   299		return id
   300	}
   301	
   302	func Index(t *testing.T, initIdx func() *index.Index) {
   303		ctx := context.Background()
   304		oldLocal := time.Local
   305		time.Local = time.UTC
   306		defer func() { time.Local = oldLocal }()
   307	
   308		id := NewIndexDeps(initIdx())
   309		id.Fataler = t
   310		defer id.DumpIndex(t)
   311		pn := id.NewPermanode()
   312		t.Logf("uploaded permanode %q", pn)
   313		br1 := id.SetAttribute(pn, "tag", "foo1")
   314		br1Time := id.LastTime()
   315		t.Logf("set attribute %q", br1)
   316		br2 := id.SetAttribute(pn, "tag", "foo2")
   317		br2Time := id.LastTime()
   318		t.Logf("set attribute %q", br2)
   319		rootClaim := id.SetAttribute(pn, "camliRoot", "rootval")
   320		rootClaimTime := id.LastTime()
   321		t.Logf("set attribute %q", rootClaim)
   322	
   323		pnChild := id.NewPermanode()
   324		id.SetAttribute(pnChild, "unindexed", "lost in time and space")
   325		br3 := id.SetAttribute(pnChild, "tag", "bar")
   326		br3Time := id.LastTime()
   327		t.Logf("set attribute %q", br3)
   328		memberRef := id.AddAttribute(pn, "camliMember", pnChild.String())
   329		t.Logf("add-attribute claim %q points to member permanode %q", memberRef, pnChild)
   330		memberRefTime := id.LastTime()
   331	
   332		// TODO(bradfitz): add EXIF tests here, once that stuff is ready.
   333		if false {
   334			camliRootPath, err := osutil.GoPackagePath("perkeep.org")
   335			if err != nil {
   336				t.Fatal("Package perkeep.org not found in $GOPATH or $GOPATH not defined")
   337			}
   338			for i := 1; i <= 8; i++ {
   339				fileBase := fmt.Sprintf("f%d-exif.jpg", i)
   340				fileName := filepath.Join(camliRootPath, "pkg", "images", "testdata", fileBase)
   341				contents, err := os.ReadFile(fileName)
   342				if err != nil {
   343					t.Fatal(err)
   344				}
   345				id.UploadFile(fileBase, string(contents), noTime)
   346			}
   347		}
   348	
   349		// Upload some files.
   350		var jpegFileRef, exifFileRef, exifWholeRef, badExifWholeRef, nanExifWholeRef, mediaFileRef, mediaWholeRef, heicEXIFWholeRef blob.Ref
   351		{
   352			camliRootPath, err := osutil.GoPackagePath("perkeep.org")
   353			if err != nil {
   354				t.Fatal("Package perkeep.org not found in $GOPATH or $GOPATH not defined")
   355			}
   356			uploadFile := func(file string, modTime time.Time) (fileRef, wholeRef blob.Ref) {
   357				fileName := filepath.Join(camliRootPath, "pkg", "index", "indextest", "testdata", file)
   358				contents, err := os.ReadFile(fileName)
   359				if err != nil {
   360					t.Fatal(err)
   361				}
   362				fileRef, wholeRef = id.UploadFile(file, string(contents), modTime)
   363				return
   364			}
   365			jpegFileRef, _ = uploadFile("dude.jpg", noTime)
   366			exifFileRef, exifWholeRef = uploadFile("dude-exif.jpg", time.Unix(1361248796, 0))
   367			_, badExifWholeRef = uploadFile("bad-exif.jpg", time.Unix(1361248796, 0))
   368			_, nanExifWholeRef = uploadFile("nan-exif.jpg", time.Unix(1361248796, 0))
   369			mediaFileRef, mediaWholeRef = uploadFile("0s.mp3", noTime)
   370			_, heicEXIFWholeRef = uploadFile("black-seattle-truncated.heic", time.Unix(1361248796, 0))
   371		}
   372	
   373		// Upload the dir containing the previous files.
   374		imagesDirRef := id.UploadDir(
   375			"testdata",
   376			[]blob.Ref{jpegFileRef, exifFileRef, mediaFileRef},
   377			time.Now(),
   378		)
   379	
   380		lastPermanodeMutation := id.LastTime()
   381	
   382		key := "signerkeyid:sha224-a794846212ff67acdd00c6b90eee492baf674d41da8a621d2e8042dd"
   383		if g, e := id.Get(key), "2931A67C26F5ABDA"; g != e {
   384			t.Fatalf("%q = %q, want %q", key, g, e)
   385		}
   386	
   387		key = "imagesize|" + jpegFileRef.String()
   388		if g, e := id.Get(key), "50|100"; g != e {
   389			t.Errorf("JPEG dude.jpg key %q = %q; want %q", key, g, e)
   390		}
   391	
   392		key = "filetimes|" + jpegFileRef.String()
   393		if g, e := id.Get(key), ""; g != e {
   394			t.Errorf("JPEG dude.jpg key %q = %q; want %q", key, g, e)
   395		}
   396	
   397		key = "filetimes|" + exifFileRef.String()
   398		if g, e := id.Get(key), "2013-02-18T01%3A11%3A20Z%2C2013-02-19T04%3A39%3A56Z"; g != e {
   399			t.Errorf("EXIF dude-exif.jpg key %q = %q; want %q", key, g, e)
   400		}
   401	
   402		key = "exifgps|" + exifWholeRef.String()
   403		// Test that small values aren't printed as scientific notation.
   404		if g, e := id.Get(key), "-0.0000010|-120.0000000"; g != e {
   405			t.Errorf("EXIF dude-exif.jpg key %q = %q; want %q", key, g, e)
   406		}
   407	
   408		// Check that indexer ignores exif lat/fields that are out of bounds.
   409		key = "exifgps|" + badExifWholeRef.String()
   410		if g, e := id.Get(key), ""; g != e {
   411			t.Errorf("EXIF bad-exif.jpg key %q = %q; want %q", key, g, e)
   412		}
   413	
   414		// Check that indexer ignores NaN exif lat/long.
   415		key = "exifgps|" + nanExifWholeRef.String()
   416		if g, e := id.Get(key), ""; g != e {
   417			t.Errorf("EXIF nan-exif.jpg key %q = %q; want %q", key, g, e)
   418		}
   419	
   420		// Check that we can read EXIF from HEIC files too
   421		key = "exifgps|" + heicEXIFWholeRef.String()
   422		if g, e := id.Get(key), "47.6496056|-122.3512806"; g != e {
   423			t.Errorf("EXIF black-seattle-truncated.heic key %q = %q; want %q", key, g, e)
   424		}
   425	
   426		key = "have:" + pn.String()
   427		pnSizeStr := strings.TrimSuffix(id.Get(key), "|indexed")
   428		if pnSizeStr == "" {
   429			t.Fatalf("missing key %q", key)
   430		}
   431	
   432		key = "meta:" + pn.String()
   433		if g, e := id.Get(key), pnSizeStr+"|application/json; camliType=permanode"; g != e {
   434			t.Errorf("key %q = %q, want %q", key, g, e)
   435		}
   436	
   437		key = "recpn|2931A67C26F5ABDA|rt7988-88-71T98:67:62.999876543Z|" + br1.String()
   438		if g, e := id.Get(key), pn.String(); g != e {
   439			t.Fatalf("%q = %q, want %q (permanode)", key, g, e)
   440		}
   441	
   442		key = "recpn|2931A67C26F5ABDA|rt7988-88-71T98:67:61.999876543Z|" + br2.String()
   443		if g, e := id.Get(key), pn.String(); g != e {
   444			t.Fatalf("%q = %q, want %q (permanode)", key, g, e)
   445		}
   446	
   447		key = fmt.Sprintf("edgeback|%s|%s|%s", pnChild, pn, memberRef)
   448		if g, e := id.Get(key), "permanode|"; g != e {
   449			t.Fatalf("edgeback row %q = %q, want %q", key, g, e)
   450		}
   451	
   452		mediaTests := []struct {
   453			prop, exp string
   454		}{
   455			{"title", "Zero Seconds"},
   456			{"artist", "Test Artist"},
   457			{"album", "Test Album"},
   458			{"genre", "(20)Alternative"},
   459			{"musicbrainzalbumid", "00000000-0000-0000-0000-000000000000"},
   460			{"year", "1992"},
   461			{"track", "1"},
   462			{"disc", "2"},
   463			{"mediaref", "sha224-c4ebd5b557419a68ba6e0af716deeb9196e71e0dca65a7f805e3f723"},
   464			{"durationms", "26"},
   465		}
   466		for _, tt := range mediaTests {
   467			key = fmt.Sprintf("mediatag|%s|%s", mediaWholeRef.String(), tt.prop)
   468			if g, _ := url.QueryUnescape(id.Get(key)); g != tt.exp {
   469				t.Errorf("0s.mp3 key %q = %q; want %q", key, g, tt.exp)
   470			}
   471		}
   472	
   473		// PermanodeOfSignerAttrValue
   474		{
   475			gotPN, err := id.Index.PermanodeOfSignerAttrValue(ctx, id.SignerBlobRef, "camliRoot", "rootval")
   476			if err != nil {
   477				t.Fatalf("id.Index.PermanodeOfSignerAttrValue = %v", err)
   478			}
   479			if gotPN.String() != pn.String() {
   480				t.Errorf("id.Index.PermanodeOfSignerAttrValue = %q, want %q", gotPN, pn)
   481			}
   482			_, err = id.Index.PermanodeOfSignerAttrValue(ctx, id.SignerBlobRef, "camliRoot", "MISSING")
   483			if err == nil {
   484				t.Errorf("expected an error from PermanodeOfSignerAttrValue on missing value")
   485			}
   486		}
   487	
   488		// SearchPermanodesWithAttr - match attr type "tag" and value "foo1"
   489		{
   490			ch := make(chan blob.Ref, 10)
   491			req := &camtypes.PermanodeByAttrRequest{
   492				Signer:    id.SignerBlobRef,
   493				Attribute: "tag",
   494				Query:     "foo1",
   495			}
   496			err := id.Index.SearchPermanodesWithAttr(ctx, ch, req)
   497			if err != nil {
   498				t.Fatalf("SearchPermanodesWithAttr = %v", err)
   499			}
   500			var got []blob.Ref
   501			for r := range ch {
   502				got = append(got, r)
   503			}
   504			want := []blob.Ref{pn}
   505			if len(got) < 1 || got[0].String() != want[0].String() {
   506				t.Errorf("id.Index.SearchPermanodesWithAttr gives %q, want %q", got, want)
   507			}
   508		}
   509	
   510		// SearchPermanodesWithAttr - match all with attr type "tag"
   511		{
   512			ch := make(chan blob.Ref, 10)
   513			req := &camtypes.PermanodeByAttrRequest{
   514				Signer:    id.SignerBlobRef,
   515				Attribute: "tag",
   516			}
   517			err := id.Index.SearchPermanodesWithAttr(ctx, ch, req)
   518			if err != nil {
   519				t.Fatalf("SearchPermanodesWithAttr = %v", err)
   520			}
   521			var got []blob.Ref
   522			for r := range ch {
   523				got = append(got, r)
   524			}
   525			want := []blob.Ref{pn, pnChild}
   526			if len(got) != len(want) {
   527				t.Errorf("SearchPermanodesWithAttr results differ.\n got: %q\nwant: %q",
   528					got, want)
   529			}
   530			for _, w := range want {
   531				found := false
   532				for _, g := range got {
   533					if g.String() == w.String() {
   534						found = true
   535						break
   536					}
   537				}
   538				if !found {
   539					t.Errorf("SearchPermanodesWithAttr: %v was not found.\n", w)
   540				}
   541			}
   542		}
   543	
   544		// SearchPermanodesWithAttr - match none with attr type "tag=nosuchtag"
   545		{
   546			ch := make(chan blob.Ref, 10)
   547			req := &camtypes.PermanodeByAttrRequest{
   548				Signer:    id.SignerBlobRef,
   549				Attribute: "tag",
   550				Query:     "nosuchtag",
   551			}
   552			err := id.Index.SearchPermanodesWithAttr(ctx, ch, req)
   553			if err != nil {
   554				t.Fatalf("SearchPermanodesWithAttr = %v", err)
   555			}
   556			var got []blob.Ref
   557			for r := range ch {
   558				got = append(got, r)
   559			}
   560			want := []blob.Ref{}
   561			if len(got) != len(want) {
   562				t.Errorf("SearchPermanodesWithAttr results differ.\n got: %q\nwant: %q",
   563					got, want)
   564			}
   565		}
   566		// SearchPermanodesWithAttr - error for unindexed attr
   567		{
   568			ch := make(chan blob.Ref, 10)
   569			req := &camtypes.PermanodeByAttrRequest{
   570				Signer:    id.SignerBlobRef,
   571				Attribute: "unindexed",
   572			}
   573			err := id.Index.SearchPermanodesWithAttr(ctx, ch, req)
   574			if err == nil {
   575				t.Fatalf("SearchPermanodesWithAttr with unindexed attribute should return an error")
   576			}
   577		}
   578	
   579		// Delete value "pony" of type "title" (which does not actually exist) for pn
   580		br4 := id.DelAttribute(pn, "title", "pony")
   581		br4Time := id.LastTime()
   582		// and verify it is not found when searching by attr
   583		{
   584			ch := make(chan blob.Ref, 10)
   585			req := &camtypes.PermanodeByAttrRequest{
   586				Signer:    id.SignerBlobRef,
   587				Attribute: "title",
   588				Query:     "pony",
   589			}
   590			err := id.Index.SearchPermanodesWithAttr(ctx, ch, req)
   591			if err != nil {
   592				t.Fatalf("SearchPermanodesWithAttr = %v", err)
   593			}
   594			var got []blob.Ref
   595			for r := range ch {
   596				got = append(got, r)
   597			}
   598			want := []blob.Ref{}
   599			if len(got) != len(want) {
   600				t.Errorf("SearchPermanodesWithAttr results differ.\n got: %q\nwant: %q",
   601					got, want)
   602			}
   603		}
   604	
   605		// GetRecentPermanodes
   606		{
   607			verify := func(prefix string, want []camtypes.RecentPermanode, before time.Time) {
   608				ch := make(chan camtypes.RecentPermanode, 10) // expect 2 results, but maybe more if buggy.
   609				err := id.Index.GetRecentPermanodes(ctx, ch, id.SignerBlobRef, 50, before)
   610				if err != nil {
   611					t.Fatalf("[%s] GetRecentPermanodes = %v", prefix, err)
   612				}
   613				got := []camtypes.RecentPermanode{}
   614				for r := range ch {
   615					got = append(got, r)
   616				}
   617				if len(got) != len(want) {
   618					t.Errorf("[%s] GetRecentPermanode results differ.\n got: %v\nwant: %v",
   619						prefix, searchResults(got), searchResults(want))
   620				}
   621				for _, w := range want {
   622					found := false
   623					for _, g := range got {
   624						if g.Equal(w) {
   625							found = true
   626							break
   627						}
   628					}
   629					if !found {
   630						t.Errorf("[%s] GetRecentPermanode: %v was not found.\n got: %v\nwant: %v",
   631							prefix, w, searchResults(got), searchResults(want))
   632					}
   633				}
   634			}
   635	
   636			want := []camtypes.RecentPermanode{
   637				{
   638					Permanode:   pn,
   639					Signer:      id.SignerBlobRef,
   640					LastModTime: br4Time,
   641				},
   642				{
   643					Permanode:   pnChild,
   644					Signer:      id.SignerBlobRef,
   645					LastModTime: br3Time,
   646				},
   647			}
   648	
   649			before := time.Time{}
   650			verify("Zero before", want, before)
   651	
   652			before = lastPermanodeMutation
   653			t.Log("lastPermanodeMutation", lastPermanodeMutation,
   654				lastPermanodeMutation.Unix())
   655			verify("Non-zero before", want[1:], before)
   656		}
   657		// GetDirMembers
   658		{
   659			ch := make(chan blob.Ref, 10) // expect 2 results
   660			err := id.Index.GetDirMembers(ctx, imagesDirRef, ch, 50)
   661			if err != nil {
   662				t.Fatalf("GetDirMembers = %v", err)
   663			}
   664			got := []blob.Ref{}
   665			for r := range ch {
   666				got = append(got, r)
   667			}
   668			want := []blob.Ref{jpegFileRef, exifFileRef, mediaFileRef}
   669			if len(got) != len(want) {
   670				t.Errorf("GetDirMembers results differ.\n got: %v\nwant: %v",
   671					got, want)
   672			}
   673			for _, w := range want {
   674				found := false
   675				for _, g := range got {
   676					if w == g {
   677						found = true
   678						break
   679					}
   680				}
   681				if !found {
   682					t.Errorf("GetDirMembers: %v was not found.", w)
   683				}
   684			}
   685		}
   686	
   687		// GetBlobMeta
   688		{
   689			meta, err := id.Index.GetBlobMeta(ctx, pn)
   690			if err != nil {
   691				t.Errorf("GetBlobMeta(%q) = %v", pn, err)
   692			} else {
   693				if e := schema.TypePermanode; meta.CamliType != e {
   694					t.Errorf("GetBlobMeta(%q) mime = %q, want %q", pn, meta.CamliType, e)
   695				}
   696				if meta.Size == 0 {
   697					t.Errorf("GetBlobMeta(%q) size is zero", pn)
   698				}
   699			}
   700			_, err = id.Index.GetBlobMeta(ctx, blob.ParseOrZero("abc-123"))
   701			if err != os.ErrNotExist {
   702				t.Errorf("GetBlobMeta(dummy blobref) = %v; want os.ErrNotExist", err)
   703			}
   704		}
   705	
   706		// AppendClaims
   707		{
   708			claims, err := id.Index.AppendClaims(ctx, nil, pn, KeyID, "")
   709			if err != nil {
   710				t.Errorf("AppendClaims = %v", err)
   711			} else {
   712				want := []camtypes.Claim{
   713					{
   714						BlobRef:   br1,
   715						Permanode: pn,
   716						Signer:    id.SignerBlobRef,
   717						Date:      br1Time.UTC(),
   718						Type:      "set-attribute",
   719						Attr:      "tag",
   720						Value:     "foo1",
   721					},
   722					{
   723						BlobRef:   br2,
   724						Permanode: pn,
   725						Signer:    id.SignerBlobRef,
   726						Date:      br2Time.UTC(),
   727						Type:      "set-attribute",
   728						Attr:      "tag",
   729						Value:     "foo2",
   730					},
   731					{
   732						BlobRef:   rootClaim,
   733						Permanode: pn,
   734						Signer:    id.SignerBlobRef,
   735						Date:      rootClaimTime.UTC(),
   736						Type:      "set-attribute",
   737						Attr:      "camliRoot",
   738						Value:     "rootval",
   739					},
   740					{
   741						BlobRef:   memberRef,
   742						Permanode: pn,
   743						Signer:    id.SignerBlobRef,
   744						Date:      memberRefTime.UTC(),
   745						Type:      "add-attribute",
   746						Attr:      "camliMember",
   747						Value:     pnChild.String(),
   748					},
   749					{
   750						BlobRef:   br4,
   751						Permanode: pn,
   752						Signer:    id.SignerBlobRef,
   753						Date:      br4Time.UTC(),
   754						Type:      "del-attribute",
   755						Attr:      "title",
   756						Value:     "pony",
   757					},
   758				}
   759				if !reflect.DeepEqual(claims, want) {
   760					t.Errorf("AppendClaims results differ.\n got: %v\nwant: %v",
   761						claims, want)
   762				}
   763			}
   764		}
   765	}
   766	
   767	func PathsOfSignerTarget(t *testing.T, initIdx func() *index.Index) {
   768		ctx := context.Background()
   769		id := NewIndexDeps(initIdx())
   770		id.Fataler = t
   771		defer id.DumpIndex(t)
   772		signer := id.SignerBlobRef
   773		pn := id.NewPermanode()
   774		t.Logf("uploaded permanode %q", pn)
   775	
   776		claim1 := id.SetAttribute(pn, "camliPath:somedir", "targ-123")
   777		claim1Time := id.LastTime().UTC()
   778		claim2 := id.SetAttribute(pn, "camliPath:with|pipe", "targ-124")
   779		claim2Time := id.LastTime().UTC()
   780		t.Logf("made path claims %q and %q", claim1, claim2)
   781	
   782		type test struct {
   783			blobref string
   784			want    int
   785		}
   786		tests := []test{
   787			{"targ-123", 1},
   788			{"targ-124", 1},
   789			{"targ-125", 0},
   790		}
   791		for _, tt := range tests {
   792			paths, err := id.Index.PathsOfSignerTarget(ctx, signer, blob.ParseOrZero(tt.blobref))
   793			if err != nil {
   794				t.Fatalf("PathsOfSignerTarget(%q): %v", tt.blobref, err)
   795			}
   796			if len(paths) != tt.want {
   797				t.Fatalf("PathsOfSignerTarget(%q) got %d results; want %d",
   798					tt.blobref, len(paths), tt.want)
   799			}
   800			if tt.blobref == "targ-123" {
   801				p := paths[0]
   802				want := fmt.Sprintf(
   803					"Path{Claim: %s, %v; Base: %s + Suffix \"somedir\" => Target targ-123}",
   804					claim1, claim1Time, pn)
   805				if g := p.String(); g != want {
   806					t.Errorf("claim wrong.\n got: %s\nwant: %s", g, want)
   807				}
   808			}
   809		}
   810		tests = []test{
   811			{"somedir", 1},
   812			{"with|pipe", 1},
   813			{"void", 0},
   814		}
   815		for _, tt := range tests {
   816			paths, err := id.Index.PathsLookup(ctx, id.SignerBlobRef, pn, tt.blobref)
   817			if err != nil {
   818				t.Fatalf("PathsLookup(%q): %v", tt.blobref, err)
   819			}
   820			if len(paths) != tt.want {
   821				t.Fatalf("PathsLookup(%q) got %d results; want %d",
   822					tt.blobref, len(paths), tt.want)
   823			}
   824			if tt.blobref == "with|pipe" {
   825				p := paths[0]
   826				want := fmt.Sprintf(
   827					"Path{Claim: %s, %s; Base: %s + Suffix \"with|pipe\" => Target targ-124}",
   828					claim2, claim2Time, pn)
   829				if g := p.String(); g != want {
   830					t.Errorf("claim wrong.\n got: %s\nwant: %s", g, want)
   831				}
   832			}
   833		}
   834	
   835		// now test deletions
   836		// Delete an existing value
   837		claim3 := id.Delete(claim2)
   838		t.Logf("claim %q deletes path claim %q", claim3, claim2)
   839		tests = []test{
   840			{"targ-123", 1},
   841			{"targ-124", 0},
   842			{"targ-125", 0},
   843		}
   844		for _, tt := range tests {
   845			signer := id.SignerBlobRef
   846			paths, err := id.Index.PathsOfSignerTarget(ctx, signer, blob.ParseOrZero(tt.blobref))
   847			if err != nil {
   848				t.Fatalf("PathsOfSignerTarget(%q): %v", tt.blobref, err)
   849			}
   850			if len(paths) != tt.want {
   851				t.Fatalf("PathsOfSignerTarget(%q) got %d results; want %d",
   852					tt.blobref, len(paths), tt.want)
   853			}
   854		}
   855		tests = []test{
   856			{"somedir", 1},
   857			{"with|pipe", 0},
   858			{"void", 0},
   859		}
   860		for _, tt := range tests {
   861			paths, err := id.Index.PathsLookup(ctx, id.SignerBlobRef, pn, tt.blobref)
   862			if err != nil {
   863				t.Fatalf("PathsLookup(%q): %v", tt.blobref, err)
   864			}
   865			if len(paths) != tt.want {
   866				t.Fatalf("PathsLookup(%q) got %d results; want %d",
   867					tt.blobref, len(paths), tt.want)
   868			}
   869		}
   870	
   871		// recreate second path, and test if the previous deletion of it
   872		// is indeed ignored.
   873		claim4 := id.Delete(claim3)
   874		t.Logf("delete claim %q deletes claim %q, which should undelete %q", claim4, claim3, claim2)
   875		tests = []test{
   876			{"targ-123", 1},
   877			{"targ-124", 1},
   878			{"targ-125", 0},
   879		}
   880		for _, tt := range tests {
   881			signer := id.SignerBlobRef
   882			paths, err := id.Index.PathsOfSignerTarget(ctx, signer, blob.ParseOrZero(tt.blobref))
   883			if err != nil {
   884				t.Fatalf("PathsOfSignerTarget(%q): %v", tt.blobref, err)
   885			}
   886			if len(paths) != tt.want {
   887				t.Fatalf("PathsOfSignerTarget(%q) got %d results; want %d",
   888					tt.blobref, len(paths), tt.want)
   889			}
   890			// and check the modtime too
   891			if tt.blobref == "targ-124" {
   892				p := paths[0]
   893				want := fmt.Sprintf(
   894					"Path{Claim: %s, %v; Base: %s + Suffix \"with|pipe\" => Target targ-124}",
   895					claim2, claim2Time, pn)
   896				if g := p.String(); g != want {
   897					t.Errorf("claim wrong.\n got: %s\nwant: %s", g, want)
   898				}
   899			}
   900		}
   901		tests = []test{
   902			{"somedir", 1},
   903			{"with|pipe", 1},
   904			{"void", 0},
   905		}
   906		for _, tt := range tests {
   907			paths, err := id.Index.PathsLookup(ctx, id.SignerBlobRef, pn, tt.blobref)
   908			if err != nil {
   909				t.Fatalf("PathsLookup(%q): %v", tt.blobref, err)
   910			}
   911			if len(paths) != tt.want {
   912				t.Fatalf("PathsLookup(%q) got %d results; want %d",
   913					tt.blobref, len(paths), tt.want)
   914			}
   915			// and check that modtime is now claim4Time
   916			if tt.blobref == "with|pipe" {
   917				p := paths[0]
   918				want := fmt.Sprintf(
   919					"Path{Claim: %s, %s; Base: %s + Suffix \"with|pipe\" => Target targ-124}",
   920					claim2, claim2Time, pn)
   921				if g := p.String(); g != want {
   922					t.Errorf("claim wrong.\n got: %s\nwant: %s", g, want)
   923				}
   924			}
   925		}
   926	}
   927	
   928	func Files(t *testing.T, initIdx func() *index.Index) {
   929		ctx := context.Background()
   930		id := NewIndexDeps(initIdx())
   931		id.Fataler = t
   932		fileTime := time.Unix(1361250375, 0)
   933		fileRef, wholeRef := id.UploadFile("foo.html", "<html>I am an html file.</html>", fileTime)
   934		t.Logf("uploaded fileref %q, wholeRef %q", fileRef, wholeRef)
   935		id.DumpIndex(t)
   936	
   937		// ExistingFileSchemas
   938		{
   939			key := fmt.Sprintf("wholetofile|%s|%s", wholeRef, fileRef)
   940			if g, e := id.Get(key), "1"; g != e {
   941				t.Fatalf("%q = %q, want %q", key, g, e)
   942			}
   943	
   944			refs, err := id.Index.ExistingFileSchemas(wholeRef)
   945			if err != nil {
   946				t.Fatalf("ExistingFileSchemas = %v", err)
   947			}
   948			want := index.WholeRefToFile{wholeRef.String(): []blob.Ref{fileRef}}
   949			if !reflect.DeepEqual(refs, want) {
   950				t.Errorf("ExistingFileSchemas got = %#v, want %#v", refs, want)
   951			}
   952		}
   953	
   954		// FileInfo
   955		{
   956			key := fmt.Sprintf("fileinfo|%s", fileRef)
   957			if g, e := id.Get(key), "31|foo.html|text%2Fhtml|sha224-35ef6689bf1b7981fb44a033ddd704ff28440779a11d927f2322bac7"; g != e {
   958				t.Fatalf("%q = %q, want %q", key, g, e)
   959			}
   960	
   961			fi, err := id.Index.GetFileInfo(ctx, fileRef)
   962			if err != nil {
   963				t.Fatalf("GetFileInfo = %v", err)
   964			}
   965			if got, want := fi.Size, int64(31); got != want {
   966				t.Errorf("Size = %d, want %d", got, want)
   967			}
   968			if got, want := fi.FileName, "foo.html"; got != want {
   969				t.Errorf("FileName = %q, want %q", got, want)
   970			}
   971			if got, want := fi.MIMEType, "text/html"; got != want {
   972				t.Errorf("MIMEType = %q, want %q", got, want)
   973			}
   974			if got, want := fi.Time, fileTime; !got.Time().Equal(want) {
   975				t.Errorf("Time = %v; want %v", got, want)
   976			}
   977			if got, want := fi.WholeRef, blob.MustParse("sha224-35ef6689bf1b7981fb44a033ddd704ff28440779a11d927f2322bac7"); got != want {
   978				t.Errorf("WholeRef = %v; want %v", got, want)
   979			}
   980		}
   981	}
   982	
   983	func EdgesTo(t *testing.T, initIdx func() *index.Index) {
   984		idx := initIdx()
   985		id := NewIndexDeps(idx)
   986		id.Fataler = t
   987		defer id.DumpIndex(t)
   988	
   989		// pn1 ---member---> pn2
   990		pn1 := id.NewPermanode()
   991		pn2 := id.NewPermanode()
   992		claim1 := id.AddAttribute(pn1, "camliMember", pn2.String())
   993	
   994		t.Logf("edge %s --> %s", pn1, pn2)
   995	
   996		// Look for pn1
   997		{
   998			edges, err := idx.EdgesTo(pn2, nil)
   999			if err != nil {
  1000				t.Fatal(err)
  1001			}
  1002			if len(edges) != 1 {
  1003				t.Fatalf("num edges = %d; want 1", len(edges))
  1004			}
  1005			wantEdge := &camtypes.Edge{
  1006				From:     pn1,
  1007				To:       pn2,
  1008				FromType: "permanode",
  1009			}
  1010			if got, want := edges[0].String(), wantEdge.String(); got != want {
  1011				t.Errorf("Wrong edge.\n GOT: %v\nWANT: %v", got, want)
  1012			}
  1013		}
  1014	
  1015		// Delete claim -> break edge relationship.
  1016		del1 := id.Delete(claim1)
  1017		t.Logf("del claim %q deletes claim %q, breaks link between p1 and p2", del1, claim1)
  1018		// test that we can't find anymore pn1 from pn2
  1019		{
  1020			edges, err := idx.EdgesTo(pn2, nil)
  1021			if err != nil {
  1022				t.Fatal(err)
  1023			}
  1024			if len(edges) != 0 {
  1025				t.Fatalf("num edges = %d; want 0", len(edges))
  1026			}
  1027		}
  1028	
  1029		// Undelete, should restore the link.
  1030		del2 := id.Delete(del1)
  1031		t.Logf("del claim %q deletes del claim %q, restores link between p1 and p2", del2, del1)
  1032		{
  1033			edges, err := idx.EdgesTo(pn2, nil)
  1034			if err != nil {
  1035				t.Fatal(err)
  1036			}
  1037			if len(edges) != 1 {
  1038				t.Fatalf("num edges = %d; want 1", len(edges))
  1039			}
  1040			wantEdge := &camtypes.Edge{
  1041				From:     pn1,
  1042				To:       pn2,
  1043				FromType: "permanode",
  1044			}
  1045			if got, want := edges[0].String(), wantEdge.String(); got != want {
  1046				t.Errorf("Wrong edge.\n GOT: %v\nWANT: %v", got, want)
  1047			}
  1048		}
  1049	}
  1050	
  1051	func Delete(t *testing.T, initIdx func() *index.Index) {
  1052		ctx := context.Background()
  1053		idx := initIdx()
  1054		id := NewIndexDeps(idx)
  1055		id.Fataler = t
  1056		defer id.DumpIndex(t)
  1057		pn1 := id.NewPermanode()
  1058		t.Logf("uploaded permanode %q", pn1)
  1059		cl1 := id.SetAttribute(pn1, "tag", "foo1")
  1060		cl1Time := id.LastTime()
  1061		t.Logf("set attribute %q", cl1)
  1062	
  1063		// delete pn1
  1064		delpn1 := id.Delete(pn1)
  1065		t.Logf("del claim %q deletes %q", delpn1, pn1)
  1066		deleted := idx.IsDeleted(pn1)
  1067		if !deleted {
  1068			t.Fatal("pn1 should be deleted")
  1069		}
  1070	
  1071		// and try to find it with SearchPermanodesWithAttr (which should not work)
  1072		{
  1073			ch := make(chan blob.Ref, 10)
  1074			req := &camtypes.PermanodeByAttrRequest{
  1075				Signer:    id.SignerBlobRef,
  1076				Attribute: "tag",
  1077				Query:     "foo1"}
  1078			err := id.Index.SearchPermanodesWithAttr(ctx, ch, req)
  1079			if err != nil {
  1080				t.Fatalf("SearchPermanodesWithAttr = %v", err)
  1081			}
  1082			var got []blob.Ref
  1083			for r := range ch {
  1084				got = append(got, r)
  1085			}
  1086			want := []blob.Ref{}
  1087			if len(got) != len(want) {
  1088				t.Errorf("id.Index.SearchPermanodesWithAttr gives %q, want %q", got, want)
  1089			}
  1090		}
  1091	
  1092		// delete pn1 again with another claim
  1093		delpn1bis := id.Delete(pn1)
  1094		t.Logf("del claim %q deletes %q a second time", delpn1bis, pn1)
  1095		deleted = idx.IsDeleted(pn1)
  1096		if !deleted {
  1097			t.Fatal("pn1 should be deleted")
  1098		}
  1099	
  1100		// verify that deleting delpn1 is not enough to make pn1 undeleted
  1101		del2 := id.Delete(delpn1)
  1102		t.Logf("delete claim %q deletes %q, which should not yet revive %q", del2, delpn1, pn1)
  1103		deleted = idx.IsDeleted(pn1)
  1104		if !deleted {
  1105			t.Fatal("pn1 should not yet be undeleted")
  1106		}
  1107		// we should not yet be able to find it again with SearchPermanodesWithAttr
  1108		{
  1109			ch := make(chan blob.Ref, 10)
  1110			req := &camtypes.PermanodeByAttrRequest{
  1111				Signer:    id.SignerBlobRef,
  1112				Attribute: "tag",
  1113				Query:     "foo1"}
  1114			err := id.Index.SearchPermanodesWithAttr(ctx, ch, req)
  1115			if err != nil {
  1116				t.Fatalf("SearchPermanodesWithAttr = %v", err)
  1117			}
  1118			var got []blob.Ref
  1119			for r := range ch {
  1120				got = append(got, r)
  1121			}
  1122			want := []blob.Ref{}
  1123			if len(got) != len(want) {
  1124				t.Errorf("id.Index.SearchPermanodesWithAttr gives %q, want %q", got, want)
  1125			}
  1126		}
  1127	
  1128		// delete delpn1bis as well -> should undelete pn1
  1129		del2bis := id.Delete(delpn1bis)
  1130		t.Logf("delete claim %q deletes %q, which should revive %q", del2bis, delpn1bis, pn1)
  1131		deleted = idx.IsDeleted(pn1)
  1132		if deleted {
  1133			t.Fatal("pn1 should be undeleted")
  1134		}
  1135		// we should now be able to find it again with SearchPermanodesWithAttr
  1136		{
  1137			ch := make(chan blob.Ref, 10)
  1138			req := &camtypes.PermanodeByAttrRequest{
  1139				Signer:    id.SignerBlobRef,
  1140				Attribute: "tag",
  1141				Query:     "foo1"}
  1142			err := id.Index.SearchPermanodesWithAttr(ctx, ch, req)
  1143			if err != nil {
  1144				t.Fatalf("SearchPermanodesWithAttr = %v", err)
  1145			}
  1146			var got []blob.Ref
  1147			for r := range ch {
  1148				got = append(got, r)
  1149			}
  1150			want := []blob.Ref{pn1}
  1151			if len(got) < 1 || got[0].String() != want[0].String() {
  1152				t.Errorf("id.Index.SearchPermanodesWithAttr gives %q, want %q", got, want)
  1153			}
  1154		}
  1155	
  1156		// Delete cl1
  1157		del3 := id.Delete(cl1)
  1158		t.Logf("del claim %q deletes claim %q", del3, cl1)
  1159		deleted = idx.IsDeleted(cl1)
  1160		if !deleted {
  1161			t.Fatal("cl1 should be deleted")
  1162		}
  1163		// we should not find anything with SearchPermanodesWithAttr
  1164		{
  1165			ch := make(chan blob.Ref, 10)
  1166			req := &camtypes.PermanodeByAttrRequest{
  1167				Signer:    id.SignerBlobRef,
  1168				Attribute: "tag",
  1169				Query:     "foo1"}
  1170			err := id.Index.SearchPermanodesWithAttr(ctx, ch, req)
  1171			if err != nil {
  1172				t.Fatalf("SearchPermanodesWithAttr = %v", err)
  1173			}
  1174			var got []blob.Ref
  1175			for r := range ch {
  1176				got = append(got, r)
  1177			}
  1178			want := []blob.Ref{}
  1179			if len(got) != len(want) {
  1180				t.Errorf("id.Index.SearchPermanodesWithAttr gives %q, want %q", got, want)
  1181			}
  1182		}
  1183		// and now check that AppendClaims finds nothing for pn
  1184		{
  1185			claims, err := id.Index.AppendClaims(ctx, nil, pn1, KeyID, "")
  1186			if err != nil {
  1187				t.Errorf("AppendClaims = %v", err)
  1188			} else {
  1189				want := []camtypes.Claim{}
  1190				if len(claims) != len(want) {
  1191					t.Errorf("id.Index.AppendClaims gives %q, want %q", claims, want)
  1192				}
  1193			}
  1194		}
  1195	
  1196		// undelete cl1
  1197		del4 := id.Delete(del3)
  1198		t.Logf("del claim %q deletes del claim %q, which should undelete %q", del4, del3, cl1)
  1199		// We should now be able to find it again with both methods
  1200		{
  1201			ch := make(chan blob.Ref, 10)
  1202			req := &camtypes.PermanodeByAttrRequest{
  1203				Signer:    id.SignerBlobRef,
  1204				Attribute: "tag",
  1205				Query:     "foo1"}
  1206			err := id.Index.SearchPermanodesWithAttr(ctx, ch, req)
  1207			if err != nil {
  1208				t.Fatalf("SearchPermanodesWithAttr = %v", err)
  1209			}
  1210			var got []blob.Ref
  1211			for r := range ch {
  1212				got = append(got, r)
  1213			}
  1214			want := []blob.Ref{pn1}
  1215			if len(got) < 1 || got[0].String() != want[0].String() {
  1216				t.Errorf("id.Index.SearchPermanodesWithAttr gives %q, want %q", got, want)
  1217			}
  1218		}
  1219		// and check that AppendClaims finds cl1, with the right modtime too
  1220		{
  1221			claims, err := id.Index.AppendClaims(ctx, nil, pn1, KeyID, "")
  1222			if err != nil {
  1223				t.Errorf("AppendClaims = %v", err)
  1224			} else {
  1225				want := []camtypes.Claim{
  1226					{
  1227						BlobRef:   cl1,
  1228						Permanode: pn1,
  1229						Signer:    id.SignerBlobRef,
  1230						Date:      cl1Time.UTC(),
  1231						Type:      "set-attribute",
  1232						Attr:      "tag",
  1233						Value:     "foo1",
  1234					},
  1235				}
  1236				if !reflect.DeepEqual(claims, want) {
  1237					t.Errorf("GetOwnerClaims results differ.\n got: %v\nwant: %v",
  1238						claims, want)
  1239				}
  1240			}
  1241		}
  1242	}
  1243	
  1244	type searchResults []camtypes.RecentPermanode
  1245	
  1246	func (s searchResults) String() string {
  1247		var buf bytes.Buffer
  1248		fmt.Fprintf(&buf, "[%d search results: ", len(s))
  1249		for _, r := range s {
  1250			fmt.Fprintf(&buf, "{BlobRef: %s, Signer: %s, LastModTime: %d}",
  1251				r.Permanode, r.Signer, r.LastModTime.Unix())
  1252		}
  1253		buf.WriteString("]")
  1254		return buf.String()
  1255	}
  1256	
  1257	func Reindex(t *testing.T, initIdx func() *index.Index) {
  1258		idx := initIdx()
  1259		id := NewIndexDeps(idx)
  1260		id.Fataler = t
  1261		defer id.DumpIndex(t)
  1262	
  1263		pn1 := id.NewPlannedPermanode("foo1")
  1264		t.Logf("uploaded permanode %q", pn1)
  1265	
  1266		// delete pn1
  1267		delpn1 := id.Delete(pn1)
  1268		t.Logf("del claim %q deletes %q", delpn1, pn1)
  1269		if deleted := idx.IsDeleted(pn1); !deleted {
  1270			t.Fatal("pn1 should be deleted")
  1271		}
  1272	
  1273		if err := id.Index.Reindex(); err != nil {
  1274			t.Fatalf("reindexing failed: %v", err)
  1275		}
  1276	
  1277		if deleted := idx.IsDeleted(pn1); !deleted {
  1278			t.Fatal("pn1 should be deleted after reindexing")
  1279		}
  1280	}
  1281	
  1282	type enumArgs struct {
  1283		ctx   context.Context
  1284		dest  chan blob.SizedRef
  1285		after string
  1286		limit int
  1287	}
  1288	
  1289	func checkEnumerate(idx *index.Index, want []blob.SizedRef, args *enumArgs) error {
  1290		if args == nil {
  1291			args = &enumArgs{}
  1292		}
  1293		if args.ctx == nil {
  1294			args.ctx = context.TODO()
  1295		}
  1296		if args.dest == nil {
  1297			args.dest = make(chan blob.SizedRef)
  1298		}
  1299		if args.limit == 0 {
  1300			args.limit = 5000
  1301		}
  1302		errCh := make(chan error)
  1303		go func() {
  1304			errCh <- idx.EnumerateBlobs(args.ctx, args.dest, args.after, args.limit)
  1305		}()
  1306		for k, sbr := range want {
  1307			got, ok := <-args.dest
  1308			if !ok {
  1309				return fmt.Errorf("could not enumerate blob %d", k)
  1310			}
  1311			if got != sbr {
  1312				return fmt.Errorf("enumeration %d: got %v, wanted %v", k, got, sbr)
  1313			}
  1314		}
  1315		_, ok := <-args.dest
  1316		if ok {
  1317			return errors.New("chan was not closed after enumeration")
  1318		}
  1319		return <-errCh
  1320	}
  1321	
  1322	func checkStat(idx *index.Index, want []blob.SizedRef) error {
  1323		pos := make(map[blob.Ref]int) // wanted ref => its position in want
  1324		need := make(map[blob.Ref]bool)
  1325		for i, sb := range want {
  1326			pos[sb.Ref] = i
  1327			need[sb.Ref] = true
  1328		}
  1329	
  1330		input := make([]blob.Ref, len(want))
  1331		for _, sbr := range want {
  1332			input = append(input, sbr.Ref)
  1333		}
  1334		err := idx.StatBlobs(context.Background(), input, func(sb blob.SizedRef) error {
  1335			if !sb.Valid() {
  1336				return errors.New("StatBlobs func called with invalid/zero blob.SizedRef")
  1337			}
  1338			wantPos, ok := pos[sb.Ref]
  1339			if !ok {
  1340				return fmt.Errorf("StatBlobs func called with unrequested ref %v (size %d)", sb.Ref, sb.Size)
  1341			}
  1342			if !need[sb.Ref] {
  1343				return fmt.Errorf("StatBlobs func called with ref %v multiple times", sb.Ref)
  1344			}
  1345			delete(need, sb.Ref)
  1346			w := want[wantPos]
  1347			if sb != w {
  1348				return fmt.Errorf("StatBlobs returned %v; want %v", sb, w)
  1349			}
  1350			return nil
  1351		})
  1352		if err != nil {
  1353			return err
  1354		}
  1355		for br := range need {
  1356			return fmt.Errorf("didn't get stat result for %v", br)
  1357		}
  1358		return nil
  1359	}
  1360	
  1361	func EnumStat(t *testing.T, initIdx func() *index.Index) {
  1362		idx := initIdx()
  1363		id := NewIndexDeps(idx)
  1364		id.Fataler = t
  1365	
  1366		type step func() error
  1367	
  1368		// so we can refer to the added permanodes without using hardcoded blobRefs
  1369		added := make(map[string]blob.Ref)
  1370	
  1371		stepAdd := func(contents string) step { // add the blob
  1372			return func() error {
  1373				pn := id.NewPlannedPermanode(contents)
  1374				t.Logf("uploaded permanode %q", pn)
  1375				added[contents] = pn
  1376				return nil
  1377			}
  1378		}
  1379	
  1380		stepEnumCheck := func(want []blob.SizedRef, args *enumArgs) step { // check the blob
  1381			return func() error {
  1382				if err := checkEnumerate(idx, want, args); err != nil {
  1383					return err
  1384				}
  1385				return nil
  1386			}
  1387		}
  1388	
  1389		missingBlob := blob.MustParse("sha224-00000000000000000000000000000000000000000000000000000000")
  1390		stepDelete := func(toDelete blob.Ref) step {
  1391			return func() error {
  1392				del := id.Delete(missingBlob)
  1393				t.Logf("added del claim %v to delete %v", del, toDelete)
  1394				return nil
  1395			}
  1396		}
  1397	
  1398		stepStatCheck := func(want []blob.SizedRef) step {
  1399			return func() error {
  1400				if err := checkStat(idx, want); err != nil {
  1401					return err
  1402				}
  1403				return nil
  1404			}
  1405		}
  1406	
  1407		for _, v := range []string{
  1408			"foo",
  1409			"barr",
  1410			"bazzz",
  1411		} {
  1412			stepAdd(v)()
  1413		}
  1414		foo := blob.SizedRef{ // sha224-c4c574a741e1728662095a720f8a98158706d9fe4fc5d565cca77f61
  1415			Ref:  blob.MustParse(added["foo"].String()),
  1416			Size: 552,
  1417		}
  1418		bar := blob.SizedRef{ // sha224-8254816368fb26b5b0f58b312c39e537ed93cbfe04db4f14e7559f93;
  1419			Ref:  blob.MustParse(added["barr"].String()),
  1420			Size: 553,
  1421		}
  1422		baz := blob.SizedRef{ // sha224-fee45fffbf949219eb4db6f393dd206a5371709fbf093218e6f272ec;
  1423			Ref:  blob.MustParse(added["bazzz"].String()),
  1424			Size: 554,
  1425		}
  1426		delMissing := blob.SizedRef{ // sha224-7fcb38ca16606814881678d30874f1b59953bc522570be8645da1d14
  1427			Ref:  blob.MustParse("sha224-7fcb38ca16606814881678d30874f1b59953bc522570be8645da1d14"),
  1428			Size: 685,
  1429		}
  1430	
  1431		if err := stepEnumCheck([]blob.SizedRef{bar, foo, baz}, nil)(); err != nil {
  1432			t.Fatalf("first enum, testing order: %v", err)
  1433		}
  1434	
  1435		// Now again, but skipping bar's blob
  1436		if err := stepEnumCheck([]blob.SizedRef{foo, baz},
  1437			&enumArgs{
  1438				after: added["barr"].String(),
  1439			},
  1440		)(); err != nil {
  1441			t.Fatalf("second enum, testing skipping with after: %v", err)
  1442		}
  1443	
  1444		// Now add a delete claim with a missing dep, which should add an "have" row in the old format,
  1445		// i.e. without the "|indexed" suffix. So we can test if we're still compatible with old rows.
  1446		stepDelete(missingBlob)()
  1447		if err := stepEnumCheck([]blob.SizedRef{delMissing, bar, foo, baz}, nil)(); err != nil {
  1448			t.Fatalf("third enum, testing old \"have\" row compat: %v", err)
  1449		}
  1450	
  1451		if err := stepStatCheck([]blob.SizedRef{delMissing, foo, bar, baz})(); err != nil {
  1452			t.Fatalf("stat check: %v", err)
  1453		}
  1454	}
  1455	
  1456	// MustNew wraps index.New and fails with a Fatal error on t if New
  1457	// returns an error.
  1458	func MustNew(t *testing.T, s sorted.KeyValue) *index.Index {
  1459		ix, err := index.New(s)
  1460		if err != nil {
  1461			t.Fatalf("Error creating index: %v", err)
  1462		}
  1463		return ix
  1464	}