Home Download Docs Code Community
     1	/*
     2	Copyright 2014 The Perkeep Authors
     3	
     4	Licensed under the Apache License, Version 2.0 (the "License");
     5	you may not use this file except in compliance with the License.
     6	You may obtain a copy of the License at
     7	
     8	     http://www.apache.org/licenses/LICENSE-2.0
     9	
    10	Unless required by applicable law or agreed to in writing, software
    11	distributed under the License is distributed on an "AS IS" BASIS,
    12	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13	See the License for the specific language governing permissions and
    14	limitations under the License.
    15	*/
    16	
    17	package search
    18	
    19	import (
    20		"bytes"
    21		"context"
    22		"encoding/json"
    23		"fmt"
    24		"log"
    25		"net/http"
    26		"net/url"
    27		"os"
    28		"sort"
    29		"strings"
    30		"sync"
    31		"time"
    32	
    33		"go4.org/syncutil"
    34		"go4.org/types"
    35	
    36		"perkeep.org/internal/httputil"
    37		"perkeep.org/pkg/blob"
    38		"perkeep.org/pkg/schema"
    39		"perkeep.org/pkg/types/camtypes"
    40	)
    41	
    42	func (sh *Handler) serveDescribe(rw http.ResponseWriter, req *http.Request) {
    43		defer httputil.RecoverJSON(rw, req)
    44		var dr DescribeRequest
    45		dr.fromHTTP(req)
    46		ctx := context.TODO()
    47	
    48		res, err := sh.Describe(ctx, &dr)
    49		if err != nil {
    50			httputil.ServeJSONError(rw, err)
    51			return
    52		}
    53		httputil.ReturnJSON(rw, res)
    54	}
    55	
    56	const verboseDescribe = false
    57	
    58	// Describe returns a response for the given describe request. It acquires RLock
    59	// on the Handler's index.
    60	func (sh *Handler) Describe(ctx context.Context, dr *DescribeRequest) (dres *DescribeResponse, err error) {
    61		sh.index.RLock()
    62		defer sh.index.RUnlock()
    63	
    64		return sh.DescribeLocked(ctx, dr)
    65	}
    66	
    67	// DescribeLocked returns a response for the given describe request. It is the
    68	// caller's responsibility to lock the search handler's index.
    69	func (sh *Handler) DescribeLocked(ctx context.Context, dr *DescribeRequest) (dres *DescribeResponse, err error) {
    70		if verboseDescribe {
    71			t0 := time.Now()
    72			defer func() {
    73				td := time.Since(t0)
    74				var num int
    75				if dres != nil {
    76					num = len(dres.Meta)
    77				}
    78				log.Printf("Described %d blobs in %v", num, td)
    79			}()
    80		}
    81		sh.initDescribeRequest(dr)
    82		if dr.BlobRef.Valid() {
    83			dr.StartDescribe(ctx, dr.BlobRef, dr.depth())
    84		}
    85		for _, br := range dr.BlobRefs {
    86			dr.StartDescribe(ctx, br, dr.depth())
    87		}
    88		if err := dr.expandRules(ctx); err != nil {
    89			return nil, err
    90		}
    91		metaMap, err := dr.metaMap()
    92		if err != nil {
    93			return nil, err
    94		}
    95		return &DescribeResponse{metaMap}, nil
    96	}
    97	
    98	type DescribeRequest struct {
    99		// BlobRefs are the blobs to describe. If length zero, BlobRef
   100		// is used.
   101		BlobRefs []blob.Ref `json:"blobrefs,omitempty"`
   102	
   103		// BlobRef is the blob to describe.
   104		BlobRef blob.Ref `json:"blobref,omitempty"`
   105	
   106		// Depth is the optional traversal depth to describe from the
   107		// root BlobRef. If zero, a default is used.
   108		// Depth is deprecated and will be removed. Use Rules instead.
   109		Depth int `json:"depth,omitempty"`
   110	
   111		// MaxDirChildren is the requested optional limit to the number
   112		// of children that should be fetched when describing a static
   113		// directory. If zero, a default is used.
   114		MaxDirChildren int `json:"maxDirChildren,omitempty"`
   115	
   116		// At specifies the time which we wish to see the state of
   117		// this blob.  If zero (unspecified), all claims will be
   118		// considered, otherwise, any claims after this date will not
   119		// be considered.
   120		At types.Time3339 `json:"at"`
   121	
   122		// Rules specifies a set of rules to instruct how to keep
   123		// expanding the described set. All rules are tested and
   124		// matching rules grow the response set until all rules no
   125		// longer match or internal limits are hit.
   126		Rules []*DescribeRule `json:"rules,omitempty"`
   127	
   128		// Internal details, used while loading.
   129		// Initialized by sh.initDescribeRequest.
   130		sh            *Handler
   131		mu            sync.Mutex // protects following:
   132		m             MetaMap
   133		started       map[blobrefAndDepth]bool // blobref -> true
   134		blobDesLock   map[blob.Ref]*sync.Mutex
   135		errs          map[string]error // blobref -> error
   136		resFromRule   map[*DescribeRule]map[blob.Ref]bool
   137		flatRuleCache []*DescribeRule // flattened once, by flatRules
   138	
   139		wg *sync.WaitGroup // for load requests
   140	}
   141	
   142	// Clone clones a DescribeRequest by JSON marshaling it and then unmarshaling it into a new object
   143	// (which is then also initialized with initDescribeRequest).
   144	func (dr *DescribeRequest) Clone() *DescribeRequest {
   145		marshaled, _ := json.Marshal(dr)
   146		res := new(DescribeRequest)
   147		json.Unmarshal(marshaled, res)
   148		dr.sh.initDescribeRequest(res)
   149		return res
   150	}
   151	
   152	type blobrefAndDepth struct {
   153		br    blob.Ref
   154		depth int
   155	}
   156	
   157	// Requires dr.mu is held
   158	func (dr *DescribeRequest) flatRules() []*DescribeRule {
   159		if dr.flatRuleCache == nil {
   160			dr.flatRuleCache = make([]*DescribeRule, 0)
   161			for _, rule := range dr.Rules {
   162				rule.appendToFlatCache(dr)
   163			}
   164		}
   165		return dr.flatRuleCache
   166	}
   167	
   168	func (r *DescribeRule) appendToFlatCache(dr *DescribeRequest) {
   169		dr.flatRuleCache = append(dr.flatRuleCache, r)
   170		for _, rchild := range r.Rules {
   171			rchild.parentRule = r
   172			rchild.appendToFlatCache(dr)
   173		}
   174	}
   175	
   176	// Requires dr.mu is held.
   177	func (dr *DescribeRequest) foreachResultBlob(fn func(blob.Ref)) {
   178		if dr.BlobRef.Valid() {
   179			fn(dr.BlobRef)
   180		}
   181		for _, br := range dr.BlobRefs {
   182			fn(br)
   183		}
   184		for brStr := range dr.m {
   185			if br, ok := blob.Parse(brStr); ok {
   186				fn(br)
   187			}
   188		}
   189	}
   190	
   191	// Requires dr.mu is held.
   192	func (dr *DescribeRequest) blobInitiallyRequested(br blob.Ref) bool {
   193		if dr.BlobRef.Valid() && dr.BlobRef == br {
   194			return true
   195		}
   196		for _, br1 := range dr.BlobRefs {
   197			if br == br1 {
   198				return true
   199			}
   200		}
   201		return false
   202	}
   203	
   204	type DescribeRule struct {
   205		// All non-zero 'If*' fields in the following set must match
   206		// for the rule to match:
   207	
   208		// IsResultRoot, if true, only matches if the blob was part of
   209		// the original search results, not a blob expanded later.
   210		IfResultRoot bool `json:"ifResultRoot,omitempty"`
   211	
   212		// IfCamliNodeType matches if the "camliNodeType" attribute
   213		// equals this value.
   214		IfCamliNodeType string `json:"ifCamliNodeType,omitempty"`
   215	
   216		// Attrs lists attributes to describe. A special case
   217		// is if the value ends in "*", which matches prefixes
   218		// (e.g. "camliPath:*" or "*").
   219		Attrs []string `json:"attrs,omitempty"`
   220	
   221		// Additional rules to run on the described results of Attrs.
   222		Rules []*DescribeRule `json:"rules,omitempty"`
   223	
   224		parentRule *DescribeRule
   225	}
   226	
   227	// DescribeResponse is the JSON response from $searchRoot/camli/search/describe.
   228	type DescribeResponse struct {
   229		Meta MetaMap `json:"meta"`
   230	}
   231	
   232	// A MetaMap is a map from blobref to a DescribedBlob.
   233	type MetaMap map[string]*DescribedBlob
   234	
   235	type DescribedBlob struct {
   236		Request *DescribeRequest `json:"-"`
   237	
   238		BlobRef   blob.Ref         `json:"blobRef"`
   239		CamliType schema.CamliType `json:"camliType,omitempty"`
   240		Size      int64            `json:"size,"`
   241	
   242		// if camliType "permanode"
   243		Permanode *DescribedPermanode `json:"permanode,omitempty"`
   244	
   245		// if camliType "file"
   246		File *camtypes.FileInfo `json:"file,omitempty"`
   247		// if camliType "directory"
   248		Dir *camtypes.FileInfo `json:"dir,omitempty"`
   249		// if camliType "file", and File.IsImage()
   250		Image *camtypes.ImageInfo `json:"image,omitempty"`
   251		// if camliType "file" and media file
   252		MediaTags map[string]string `json:"mediaTags,omitempty"`
   253	
   254		// if camliType "directory"
   255		DirChildren []blob.Ref `json:"dirChildren,omitempty"`
   256	
   257		// Location specifies the location of the entity referenced
   258		// by the blob.
   259		//
   260		// If camliType is "file", then location comes from the metadata
   261		// (currently Exif) metadata of the file content.
   262		//
   263		// If camliType is "permanode", then location comes
   264		// from one of the following sources:
   265		//  1. Permanode attributes "latitude" and "longitude"
   266		//  2. Referenced permanode attributes (eg. for "foursquare.com:checkin"
   267		//     its "foursquareVenuePermanode")
   268		//  3. Location in permanode camliContent file metadata
   269		// The sources are checked in this order, the location from
   270		// the first source yielding a valid result is returned.
   271		Location *camtypes.Location `json:"location,omitempty"`
   272	
   273		// Stub is set if this is not loaded, but referenced.
   274		Stub bool `json:"-"`
   275	}
   276	
   277	func (m MetaMap) Get(br blob.Ref) *DescribedBlob {
   278		if !br.Valid() {
   279			return nil
   280		}
   281		return m[br.String()]
   282	}
   283	
   284	// URLSuffixPost returns the URL suffix for POST requests.
   285	func (dr *DescribeRequest) URLSuffixPost() string {
   286		return "camli/search/describe"
   287	}
   288	
   289	// URLSuffix returns the URL suffix for GET requests.
   290	// This is deprecated.
   291	func (dr *DescribeRequest) URLSuffix() string {
   292		var buf bytes.Buffer
   293		fmt.Fprintf(&buf, "camli/search/describe?depth=%d&maxdirchildren=%d",
   294			dr.depth(), dr.maxDirChildren())
   295		for _, br := range dr.BlobRefs {
   296			buf.WriteString("&blobref=")
   297			buf.WriteString(br.String())
   298		}
   299		if len(dr.BlobRefs) == 0 && dr.BlobRef.Valid() {
   300			buf.WriteString("&blobref=")
   301			buf.WriteString(dr.BlobRef.String())
   302		}
   303		if !dr.At.IsAnyZero() {
   304			buf.WriteString("&at=")
   305			buf.WriteString(dr.At.String())
   306		}
   307		return buf.String()
   308	}
   309	
   310	// fromHTTP panics with an httputil value on failure
   311	func (dr *DescribeRequest) fromHTTP(req *http.Request) {
   312		switch {
   313		case httputil.IsGet(req):
   314			dr.fromHTTPGet(req)
   315		case req.Method == "POST":
   316			dr.fromHTTPPost(req)
   317		default:
   318			panic("Unsupported method")
   319		}
   320	}
   321	
   322	func (dr *DescribeRequest) fromHTTPPost(req *http.Request) {
   323		err := json.NewDecoder(req.Body).Decode(dr)
   324		if err != nil {
   325			panic(err)
   326		}
   327	}
   328	
   329	func (dr *DescribeRequest) fromHTTPGet(req *http.Request) {
   330		req.ParseForm()
   331		if vv := req.Form["blobref"]; len(vv) > 1 {
   332			for _, brs := range vv {
   333				if br, ok := blob.Parse(brs); ok {
   334					dr.BlobRefs = append(dr.BlobRefs, br)
   335				} else {
   336					panic(httputil.InvalidParameterError("blobref"))
   337				}
   338			}
   339		} else {
   340			dr.BlobRef = httputil.MustGetBlobRef(req, "blobref")
   341		}
   342		dr.Depth = httputil.OptionalInt(req, "depth")
   343		dr.MaxDirChildren = httputil.OptionalInt(req, "maxdirchildren")
   344		dr.At = types.ParseTime3339OrZero(req.FormValue("at"))
   345	}
   346	
   347	// PermanodeFile returns in path the blobref of the described permanode
   348	// and the blobref of its File camliContent.
   349	// If b isn't a permanode, or doesn't have a camliContent that
   350	// is a file blob, ok is false.
   351	func (b *DescribedBlob) PermanodeFile() (path []blob.Ref, fi *camtypes.FileInfo, ok bool) {
   352		if b == nil || b.Permanode == nil {
   353			return
   354		}
   355		if contentRef := b.Permanode.Attr.Get("camliContent"); contentRef != "" {
   356			if cdes := b.Request.DescribedBlobStr(contentRef); cdes != nil && cdes.File != nil {
   357				return []blob.Ref{b.BlobRef, cdes.BlobRef}, cdes.File, true
   358			}
   359		}
   360		return
   361	}
   362	
   363	// PermanodeDir returns in path the blobref of the described permanode
   364	// and the blobref of its Directory camliContent.
   365	// If b isn't a permanode, or doesn't have a camliContent that
   366	// is a directory blob, ok is false.
   367	func (b *DescribedBlob) PermanodeDir() (path []blob.Ref, fi *camtypes.FileInfo, ok bool) {
   368		if b == nil || b.Permanode == nil {
   369			return
   370		}
   371		if contentRef := b.Permanode.Attr.Get("camliContent"); contentRef != "" {
   372			if cdes := b.Request.DescribedBlobStr(contentRef); cdes != nil && cdes.Dir != nil {
   373				return []blob.Ref{b.BlobRef, cdes.BlobRef}, cdes.Dir, true
   374			}
   375		}
   376		return
   377	}
   378	
   379	func (b *DescribedBlob) DomID() string {
   380		if b == nil {
   381			return ""
   382		}
   383		return b.BlobRef.DomID()
   384	}
   385	
   386	func (b *DescribedBlob) Title() string {
   387		if b == nil {
   388			return ""
   389		}
   390		if b.Permanode != nil {
   391			if t := b.Permanode.Attr.Get("title"); t != "" {
   392				return t
   393			}
   394			if contentRef := b.Permanode.Attr.Get("camliContent"); contentRef != "" {
   395				return b.Request.DescribedBlobStr(contentRef).Title()
   396			}
   397		}
   398		if b.File != nil {
   399			return b.File.FileName
   400		}
   401		if b.Dir != nil {
   402			return b.Dir.FileName
   403		}
   404		return ""
   405	}
   406	
   407	func (b *DescribedBlob) Description() string {
   408		if b == nil {
   409			return ""
   410		}
   411		if b.Permanode != nil {
   412			return b.Permanode.Attr.Get("description")
   413		}
   414		return ""
   415	}
   416	
   417	// Members returns all of b's children, as given by b's camliMember and camliPath:*
   418	// attributes. Only the first entry for a given camliPath attribute is used.
   419	func (b *DescribedBlob) Members() []*DescribedBlob {
   420		if b == nil {
   421			return nil
   422		}
   423		m := make([]*DescribedBlob, 0)
   424		if b.Permanode != nil {
   425			for _, bstr := range b.Permanode.Attr["camliMember"] {
   426				if br, ok := blob.Parse(bstr); ok {
   427					m = append(m, b.PeerBlob(br))
   428				}
   429			}
   430			for k, bstrs := range b.Permanode.Attr {
   431				if strings.HasPrefix(k, "camliPath:") && len(bstrs) > 0 {
   432					if br, ok := blob.Parse(bstrs[0]); ok {
   433						m = append(m, b.PeerBlob(br))
   434					}
   435				}
   436			}
   437		}
   438		return m
   439	}
   440	
   441	func (b *DescribedBlob) DirMembers() []*DescribedBlob {
   442		if b == nil || b.Dir == nil || len(b.DirChildren) == 0 {
   443			return nil
   444		}
   445	
   446		m := make([]*DescribedBlob, 0)
   447		for _, br := range b.DirChildren {
   448			m = append(m, b.PeerBlob(br))
   449		}
   450		return m
   451	}
   452	
   453	func (b *DescribedBlob) ContentRef() (br blob.Ref, ok bool) {
   454		if b != nil && b.Permanode != nil {
   455			if cref := b.Permanode.Attr.Get("camliContent"); cref != "" {
   456				return blob.Parse(cref)
   457			}
   458		}
   459		return
   460	}
   461	
   462	// DescribedBlobStr when given a blobref string returns a Description
   463	// or nil.  dr may be nil itself.
   464	func (dr *DescribeRequest) DescribedBlobStr(blobstr string) *DescribedBlob {
   465		if dr == nil {
   466			return nil
   467		}
   468		dr.mu.Lock()
   469		defer dr.mu.Unlock()
   470		return dr.m[blobstr]
   471	}
   472	
   473	// PeerBlob returns a DescribedBlob for the provided blobref.
   474	//
   475	// Unlike DescribedBlobStr, the returned DescribedBlob is never nil.
   476	//
   477	// If the blob was never loaded along with the the receiver (or if the
   478	// receiver is nil), a stub DescribedBlob is returned with its Stub
   479	// field set true.
   480	func (b *DescribedBlob) PeerBlob(br blob.Ref) *DescribedBlob {
   481		if b.Request == nil {
   482			return &DescribedBlob{BlobRef: br, Stub: true}
   483		}
   484		b.Request.mu.Lock()
   485		defer b.Request.mu.Unlock()
   486		return b.peerBlob(br)
   487	}
   488	
   489	// version of PeerBlob when b.Request.mu is already held.
   490	func (b *DescribedBlob) peerBlob(br blob.Ref) *DescribedBlob {
   491		if peer, ok := b.Request.m[br.String()]; ok {
   492			return peer
   493		}
   494		return &DescribedBlob{Request: b.Request, BlobRef: br, Stub: true}
   495	}
   496	
   497	type DescribedPermanode struct {
   498		Attr    url.Values `json:"attr"` // a map[string][]string
   499		ModTime time.Time  `json:"modtime,omitempty"`
   500	}
   501	
   502	// IsContainer returns whether the permanode has either named ("camliPath:"-prefixed) or unnamed
   503	// ("camliMember") member attributes.
   504	func (dp *DescribedPermanode) IsContainer() bool {
   505		if members := dp.Attr["camliMember"]; len(members) > 0 {
   506			return true
   507		}
   508		for k := range dp.Attr {
   509			if strings.HasPrefix(k, "camliPath:") {
   510				return true
   511			}
   512		}
   513		return false
   514	}
   515	
   516	// NewDescribeRequest returns a new DescribeRequest holding the state
   517	// of blobs and their summarized descriptions.  Use DescribeBlob
   518	// one or more times before calling Result.
   519	func (sh *Handler) NewDescribeRequest() *DescribeRequest {
   520		dr := new(DescribeRequest)
   521		sh.initDescribeRequest(dr)
   522		return dr
   523	}
   524	
   525	func (sh *Handler) initDescribeRequest(req *DescribeRequest) {
   526		if req.sh != nil {
   527			panic("already initialized")
   528		}
   529		req.sh = sh
   530		req.m = make(MetaMap)
   531		req.errs = make(map[string]error)
   532		req.wg = new(sync.WaitGroup)
   533	}
   534	
   535	type DescribeError map[string]error
   536	
   537	func (de DescribeError) Error() string {
   538		var buf bytes.Buffer
   539		for b, err := range de {
   540			fmt.Fprintf(&buf, "%s: %v; ", b, err)
   541		}
   542		return fmt.Sprintf("Errors (%d) describing blobs: %s", len(de), buf.String())
   543	}
   544	
   545	// Result waits for all outstanding lookups to complete and
   546	// returns the map of blobref (strings) to their described
   547	// results. The returned error is non-nil if any errors
   548	// occurred, and will be of type DescribeError.
   549	func (dr *DescribeRequest) Result() (desmap map[string]*DescribedBlob, err error) {
   550		dr.wg.Wait()
   551		// TODO: set "done" / locked flag, so no more DescribeBlob can
   552		// be called.
   553		if len(dr.errs) > 0 {
   554			return dr.m, DescribeError(dr.errs)
   555		}
   556		return dr.m, nil
   557	}
   558	
   559	func (dr *DescribeRequest) depth() int {
   560		if dr.Depth > 0 {
   561			return dr.Depth
   562		}
   563		return 1
   564	}
   565	
   566	func (dr *DescribeRequest) maxDirChildren() int {
   567		return sanitizeNumResults(dr.MaxDirChildren)
   568	}
   569	
   570	func (dr *DescribeRequest) metaMap() (map[string]*DescribedBlob, error) {
   571		dr.wg.Wait()
   572		dr.mu.Lock()
   573		defer dr.mu.Unlock()
   574		for k, err := range dr.errs {
   575			// TODO: include all?
   576			return nil, fmt.Errorf("error populating %s: %v", k, err)
   577		}
   578		m := make(map[string]*DescribedBlob)
   579		for k, desb := range dr.m {
   580			m[k] = desb
   581		}
   582		return m, nil
   583	}
   584	
   585	func (dr *DescribeRequest) describedBlob(b blob.Ref) *DescribedBlob {
   586		dr.mu.Lock()
   587		defer dr.mu.Unlock()
   588		bs := b.String()
   589		if des, ok := dr.m[bs]; ok {
   590			return des
   591		}
   592		des := &DescribedBlob{Request: dr, BlobRef: b}
   593		dr.m[bs] = des
   594		return des
   595	}
   596	
   597	func (dr *DescribeRequest) DescribeSync(ctx context.Context, br blob.Ref) (*DescribedBlob, error) {
   598		dr.StartDescribe(ctx, br, 1)
   599		res, err := dr.Result()
   600		if err != nil {
   601			return nil, err
   602		}
   603		return res[br.String()], nil
   604	}
   605	
   606	// StartDescribe starts a lookup of br, down to the provided depth.
   607	// It returns immediately. One should call Result to wait for the description to
   608	// be completed.
   609	func (dr *DescribeRequest) StartDescribe(ctx context.Context, br blob.Ref, depth int) {
   610		if depth <= 0 {
   611			return
   612		}
   613		dr.mu.Lock()
   614		defer dr.mu.Unlock()
   615		if dr.blobDesLock == nil {
   616			dr.blobDesLock = make(map[blob.Ref]*sync.Mutex)
   617		}
   618		desBlobMu, ok := dr.blobDesLock[br]
   619		if !ok {
   620			desBlobMu = new(sync.Mutex)
   621			dr.blobDesLock[br] = desBlobMu
   622		}
   623		if dr.started == nil {
   624			dr.started = make(map[blobrefAndDepth]bool)
   625		}
   626		key := blobrefAndDepth{br, depth}
   627		if dr.started[key] {
   628			return
   629		}
   630		dr.started[key] = true
   631		dr.wg.Add(1)
   632		go func() {
   633			defer dr.wg.Done()
   634			desBlobMu.Lock()
   635			defer desBlobMu.Unlock()
   636			dr.doDescribe(ctx, br, depth)
   637		}()
   638	}
   639	
   640	// requires dr.mu be held.
   641	func (r *DescribeRule) newMatches(br blob.Ref, dr *DescribeRequest) (brs []blob.Ref) {
   642		if r.IfResultRoot {
   643			if !dr.blobInitiallyRequested(br) {
   644				return nil
   645			}
   646		}
   647		if r.parentRule != nil {
   648			if _, ok := dr.resFromRule[r.parentRule][br]; !ok {
   649				return nil
   650			}
   651		}
   652		db, ok := dr.m[br.String()]
   653		if !ok || db.Permanode == nil {
   654			return nil
   655		}
   656		if t := r.IfCamliNodeType; t != "" {
   657			gotType := db.Permanode.Attr.Get("camliNodeType")
   658			if gotType != t {
   659				return nil
   660			}
   661		}
   662		for attr, vv := range db.Permanode.Attr {
   663			matches := false
   664			for _, matchAttr := range r.Attrs {
   665				if attr == matchAttr {
   666					matches = true
   667					break
   668				}
   669				if strings.HasSuffix(matchAttr, "*") && strings.HasPrefix(attr, strings.TrimSuffix(matchAttr, "*")) {
   670					matches = true
   671					break
   672				}
   673			}
   674			if !matches {
   675				continue
   676			}
   677			for _, v := range vv {
   678				if br, ok := blob.Parse(v); ok {
   679					brs = append(brs, br)
   680				}
   681			}
   682		}
   683		return brs
   684	}
   685	
   686	// dr.mu just be locked.
   687	func (dr *DescribeRequest) noteResultFromRule(rule *DescribeRule, br blob.Ref) {
   688		if dr.resFromRule == nil {
   689			dr.resFromRule = make(map[*DescribeRule]map[blob.Ref]bool)
   690		}
   691		m, ok := dr.resFromRule[rule]
   692		if !ok {
   693			m = make(map[blob.Ref]bool)
   694			dr.resFromRule[rule] = m
   695		}
   696		m[br] = true
   697	}
   698	
   699	func (dr *DescribeRequest) expandRules(ctx context.Context) error {
   700		loop := true
   701	
   702		for loop {
   703			loop = false
   704			dr.wg.Wait()
   705			dr.mu.Lock()
   706			len0 := len(dr.m)
   707			var new []blob.Ref
   708			for _, rule := range dr.flatRules() {
   709				dr.foreachResultBlob(func(br blob.Ref) {
   710					for _, nbr := range rule.newMatches(br, dr) {
   711						new = append(new, nbr)
   712						dr.noteResultFromRule(rule, nbr)
   713					}
   714				})
   715			}
   716			dr.mu.Unlock()
   717			for _, br := range new {
   718				dr.StartDescribe(ctx, br, 1)
   719			}
   720			dr.wg.Wait()
   721			dr.mu.Lock()
   722			len1 := len(dr.m)
   723			dr.mu.Unlock()
   724			loop = len0 != len1
   725		}
   726		return nil
   727	}
   728	
   729	func (dr *DescribeRequest) addError(br blob.Ref, err error) {
   730		if err == nil {
   731			return
   732		}
   733		dr.mu.Lock()
   734		defer dr.mu.Unlock()
   735		// TODO: append? meh.
   736		dr.errs[br.String()] = err
   737	}
   738	
   739	func (dr *DescribeRequest) doDescribe(ctx context.Context, br blob.Ref, depth int) {
   740		meta, err := dr.sh.index.GetBlobMeta(ctx, br)
   741		if err == os.ErrNotExist {
   742			return
   743		}
   744		if err != nil {
   745			dr.addError(br, err)
   746			return
   747		}
   748	
   749		// TODO: convert all this in terms of
   750		// DescribedBlob/DescribedPermanode/DescribedFile, not json
   751		// maps.  Then add JSON marhsallers to those types. Add tests.
   752		des := dr.describedBlob(br)
   753		if meta.CamliType != "" {
   754			des.setMIMEType("application/json; camliType=" + string(meta.CamliType))
   755		}
   756		des.Size = int64(meta.Size)
   757	
   758		switch des.CamliType {
   759		case "permanode":
   760			des.Permanode = new(DescribedPermanode)
   761			dr.populatePermanodeFields(ctx, des.Permanode, br, depth)
   762			var at time.Time
   763			if !dr.At.IsAnyZero() {
   764				at = dr.At.Time()
   765			}
   766			if loc, err := dr.sh.lh.PermanodeLocation(ctx, br, at, dr.sh.owner); err == nil {
   767				des.Location = &loc
   768			} else {
   769				if err != os.ErrNotExist {
   770					log.Printf("PermanodeLocation(permanode %s): %v", br, err)
   771				}
   772			}
   773		case "file":
   774			fi, err := dr.sh.index.GetFileInfo(ctx, br)
   775			if err != nil {
   776				if os.IsNotExist(err) {
   777					log.Printf("index.GetFileInfo(file %s) failed; index stale?", br)
   778				} else {
   779					dr.addError(br, err)
   780				}
   781				return
   782			}
   783			des.File = &fi
   784			if des.File.IsImage() {
   785				imgInfo, err := dr.sh.index.GetImageInfo(ctx, br)
   786				if err != nil {
   787					if !os.IsNotExist(err) {
   788						dr.addError(br, err)
   789					}
   790				} else {
   791					des.Image = &imgInfo
   792				}
   793			}
   794			if mediaTags, err := dr.sh.index.GetMediaTags(ctx, br); err == nil {
   795				des.MediaTags = mediaTags
   796			}
   797			if loc, err := dr.sh.index.GetFileLocation(ctx, br); err == nil {
   798				des.Location = &loc
   799			} else {
   800				if err != os.ErrNotExist {
   801					log.Printf("index.GetFileLocation(file %s): %v", br, err)
   802				}
   803			}
   804		case "directory":
   805			var g syncutil.Group
   806			g.Go(func() (err error) {
   807				fi, err := dr.sh.index.GetFileInfo(ctx, br)
   808				if os.IsNotExist(err) {
   809					log.Printf("index.GetFileInfo(directory %s) failed; index stale?", br)
   810				}
   811				if err == nil {
   812					des.Dir = &fi
   813				}
   814				return
   815			})
   816			g.Go(func() (err error) {
   817				des.DirChildren, err = dr.getDirMembers(ctx, br, depth)
   818				return
   819			})
   820			if err := g.Err(); err != nil {
   821				dr.addError(br, err)
   822			}
   823		}
   824	}
   825	
   826	func (dr *DescribeRequest) populatePermanodeFields(ctx context.Context, pi *DescribedPermanode, pn blob.Ref, depth int) {
   827		pi.Attr = make(url.Values)
   828		attr := pi.Attr
   829	
   830		claims, err := dr.sh.index.AppendClaims(ctx, nil, pn, dr.sh.owner.KeyID(), "")
   831		if err != nil {
   832			log.Printf("Error getting claims of %s: %v", pn.String(), err)
   833			dr.addError(pn, fmt.Errorf("Error getting claims of %s: %v", pn.String(), err))
   834			return
   835		}
   836	
   837		sort.Sort(camtypes.ClaimsByDate(claims))
   838	claimLoop:
   839		for _, cl := range claims {
   840			if !dr.At.IsAnyZero() {
   841				if cl.Date.After(dr.At.Time()) {
   842					continue
   843				}
   844			}
   845			switch cl.Type {
   846			default:
   847				continue
   848			case "del-attribute":
   849				if cl.Value == "" {
   850					delete(attr, cl.Attr)
   851				} else {
   852					sl := attr[cl.Attr]
   853					filtered := make([]string, 0, len(sl))
   854					for _, val := range sl {
   855						if val != cl.Value {
   856							filtered = append(filtered, val)
   857						}
   858					}
   859					attr[cl.Attr] = filtered
   860				}
   861			case "set-attribute":
   862				delete(attr, cl.Attr)
   863				fallthrough
   864			case "add-attribute":
   865				if cl.Value == "" {
   866					continue
   867				}
   868				sl, ok := attr[cl.Attr]
   869				if ok {
   870					for _, exist := range sl {
   871						if exist == cl.Value {
   872							continue claimLoop
   873						}
   874					}
   875				} else {
   876					sl = make([]string, 0, 1)
   877					attr[cl.Attr] = sl
   878				}
   879				attr[cl.Attr] = append(sl, cl.Value)
   880			}
   881			pi.ModTime = cl.Date
   882		}
   883	
   884		// Descend into any references in current attributes.
   885		for key, vals := range attr {
   886			dr.describeRefs(ctx, key, depth)
   887			for _, v := range vals {
   888				dr.describeRefs(ctx, v, depth)
   889			}
   890		}
   891	}
   892	
   893	func (dr *DescribeRequest) getDirMembers(ctx context.Context, br blob.Ref, depth int) ([]blob.Ref, error) {
   894		limit := dr.maxDirChildren()
   895		ch := make(chan blob.Ref)
   896		errch := make(chan error)
   897		go func() {
   898			errch <- dr.sh.index.GetDirMembers(ctx, br, ch, limit)
   899		}()
   900	
   901		var members []blob.Ref
   902		for child := range ch {
   903			dr.StartDescribe(ctx, child, depth)
   904			members = append(members, child)
   905		}
   906		if err := <-errch; err != nil {
   907			return nil, err
   908		}
   909		return members, nil
   910	}
   911	
   912	func (dr *DescribeRequest) describeRefs(ctx context.Context, str string, depth int) {
   913		for _, match := range blobRefPattern.FindAllString(str, -1) {
   914			if ref, ok := blob.ParseKnown(match); ok {
   915				dr.StartDescribe(ctx, ref, depth-1)
   916			}
   917		}
   918	}
   919	
   920	func (b *DescribedBlob) setMIMEType(mime string) {
   921		if strings.HasPrefix(mime, camliTypePrefix) {
   922			b.CamliType = schema.CamliType(strings.TrimPrefix(mime, camliTypePrefix))
   923		}
   924	}
Website layout inspired by memcached.
Content by the authors.