Home Download Docs Code Community
     1	/*
     2	Copyright 2014 The Perkeep Authors
     3	
     4	Licensed under the Apache License, Version 2.0 (the "License");
     5	you may not use this file except in compliance with the License.
     6	You may obtain a copy of the License at
     7	
     8	     http://www.apache.org/licenses/LICENSE-2.0
     9	
    10	Unless required by applicable law or agreed to in writing, software
    11	distributed under the License is distributed on an "AS IS" BASIS,
    12	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13	See the License for the specific language governing permissions and
    14	limitations under the License.
    15	*/
    16	
    17	package search
    18	
    19	import (
    20		"bytes"
    21		"context"
    22		"encoding/json"
    23		"fmt"
    24		"log"
    25		"net/http"
    26		"net/url"
    27		"os"
    28		"sort"
    29		"strings"
    30		"sync"
    31		"time"
    32	
    33		"go4.org/syncutil"
    34		"go4.org/types"
    35		"perkeep.org/internal/httputil"
    36		"perkeep.org/pkg/blob"
    37		"perkeep.org/pkg/types/camtypes"
    38	)
    39	
    40	func (sh *Handler) serveDescribe(rw http.ResponseWriter, req *http.Request) {
    41		defer httputil.RecoverJSON(rw, req)
    42		var dr DescribeRequest
    43		dr.fromHTTP(req)
    44		ctx := context.TODO()
    45	
    46		res, err := sh.Describe(ctx, &dr)
    47		if err != nil {
    48			httputil.ServeJSONError(rw, err)
    49			return
    50		}
    51		httputil.ReturnJSON(rw, res)
    52	}
    53	
    54	const verboseDescribe = false
    55	
    56	// Describe returns a response for the given describe request. It acquires RLock
    57	// on the Handler's index.
    58	func (sh *Handler) Describe(ctx context.Context, dr *DescribeRequest) (dres *DescribeResponse, err error) {
    59		sh.index.RLock()
    60		defer sh.index.RUnlock()
    61	
    62		return sh.DescribeLocked(ctx, dr)
    63	}
    64	
    65	// DescribeLocked returns a response for the given describe request. It is the
    66	// caller's responsibility to lock the search handler's index.
    67	func (sh *Handler) DescribeLocked(ctx context.Context, dr *DescribeRequest) (dres *DescribeResponse, err error) {
    68		if verboseDescribe {
    69			t0 := time.Now()
    70			defer func() {
    71				td := time.Since(t0)
    72				var num int
    73				if dres != nil {
    74					num = len(dres.Meta)
    75				}
    76				log.Printf("Described %d blobs in %v", num, td)
    77			}()
    78		}
    79		sh.initDescribeRequest(dr)
    80		if dr.BlobRef.Valid() {
    81			dr.StartDescribe(ctx, dr.BlobRef, dr.depth())
    82		}
    83		for _, br := range dr.BlobRefs {
    84			dr.StartDescribe(ctx, br, dr.depth())
    85		}
    86		if err := dr.expandRules(ctx); err != nil {
    87			return nil, err
    88		}
    89		metaMap, err := dr.metaMap()
    90		if err != nil {
    91			return nil, err
    92		}
    93		return &DescribeResponse{metaMap}, nil
    94	}
    95	
    96	type DescribeRequest struct {
    97		// BlobRefs are the blobs to describe. If length zero, BlobRef
    98		// is used.
    99		BlobRefs []blob.Ref `json:"blobrefs,omitempty"`
   100	
   101		// BlobRef is the blob to describe.
   102		BlobRef blob.Ref `json:"blobref,omitempty"`
   103	
   104		// Depth is the optional traversal depth to describe from the
   105		// root BlobRef. If zero, a default is used.
   106		// Depth is deprecated and will be removed. Use Rules instead.
   107		Depth int `json:"depth,omitempty"`
   108	
   109		// MaxDirChildren is the requested optional limit to the number
   110		// of children that should be fetched when describing a static
   111		// directory. If zero, a default is used.
   112		MaxDirChildren int `json:"maxDirChildren,omitempty"`
   113	
   114		// At specifies the time which we wish to see the state of
   115		// this blob.  If zero (unspecified), all claims will be
   116		// considered, otherwise, any claims after this date will not
   117		// be considered.
   118		At types.Time3339 `json:"at"`
   119	
   120		// Rules specifies a set of rules to instruct how to keep
   121		// expanding the described set. All rules are tested and
   122		// matching rules grow the response set until all rules no
   123		// longer match or internal limits are hit.
   124		Rules []*DescribeRule `json:"rules,omitempty"`
   125	
   126		// Internal details, used while loading.
   127		// Initialized by sh.initDescribeRequest.
   128		sh            *Handler
   129		mu            sync.Mutex // protects following:
   130		m             MetaMap
   131		started       map[blobrefAndDepth]bool // blobref -> true
   132		blobDesLock   map[blob.Ref]*sync.Mutex
   133		errs          map[string]error // blobref -> error
   134		resFromRule   map[*DescribeRule]map[blob.Ref]bool
   135		flatRuleCache []*DescribeRule // flattened once, by flatRules
   136	
   137		wg *sync.WaitGroup // for load requests
   138	}
   139	
   140	type blobrefAndDepth struct {
   141		br    blob.Ref
   142		depth int
   143	}
   144	
   145	// Requires dr.mu is held
   146	func (dr *DescribeRequest) flatRules() []*DescribeRule {
   147		if dr.flatRuleCache == nil {
   148			dr.flatRuleCache = make([]*DescribeRule, 0)
   149			for _, rule := range dr.Rules {
   150				rule.appendToFlatCache(dr)
   151			}
   152		}
   153		return dr.flatRuleCache
   154	}
   155	
   156	func (r *DescribeRule) appendToFlatCache(dr *DescribeRequest) {
   157		dr.flatRuleCache = append(dr.flatRuleCache, r)
   158		for _, rchild := range r.Rules {
   159			rchild.parentRule = r
   160			rchild.appendToFlatCache(dr)
   161		}
   162	}
   163	
   164	// Requires dr.mu is held.
   165	func (dr *DescribeRequest) foreachResultBlob(fn func(blob.Ref)) {
   166		if dr.BlobRef.Valid() {
   167			fn(dr.BlobRef)
   168		}
   169		for _, br := range dr.BlobRefs {
   170			fn(br)
   171		}
   172		for brStr := range dr.m {
   173			if br, ok := blob.Parse(brStr); ok {
   174				fn(br)
   175			}
   176		}
   177	}
   178	
   179	// Requires dr.mu is held.
   180	func (dr *DescribeRequest) blobInitiallyRequested(br blob.Ref) bool {
   181		if dr.BlobRef.Valid() && dr.BlobRef == br {
   182			return true
   183		}
   184		for _, br1 := range dr.BlobRefs {
   185			if br == br1 {
   186				return true
   187			}
   188		}
   189		return false
   190	}
   191	
   192	type DescribeRule struct {
   193		// All non-zero 'If*' fields in the following set must match
   194		// for the rule to match:
   195	
   196		// IsResultRoot, if true, only matches if the blob was part of
   197		// the original search results, not a blob expanded later.
   198		IfResultRoot bool `json:"ifResultRoot,omitempty"`
   199	
   200		// IfCamliNodeType matches if the "camliNodeType" attribute
   201		// equals this value.
   202		IfCamliNodeType string `json:"ifCamliNodeType,omitempty"`
   203	
   204		// Attrs lists attributes to describe. A special case
   205		// is if the value ends in "*", which matches prefixes
   206		// (e.g. "camliPath:*" or "*").
   207		Attrs []string `json:"attrs,omitempty"`
   208	
   209		// Additional rules to run on the described results of Attrs.
   210		Rules []*DescribeRule `json:"rules,omitempty"`
   211	
   212		parentRule *DescribeRule
   213	}
   214	
   215	// DescribeResponse is the JSON response from $searchRoot/camli/search/describe.
   216	type DescribeResponse struct {
   217		Meta MetaMap `json:"meta"`
   218	}
   219	
   220	// A MetaMap is a map from blobref to a DescribedBlob.
   221	type MetaMap map[string]*DescribedBlob
   222	
   223	type DescribedBlob struct {
   224		Request *DescribeRequest `json:"-"`
   225	
   226		BlobRef   blob.Ref `json:"blobRef"`
   227		CamliType string   `json:"camliType,omitempty"`
   228		Size      int64    `json:"size,"`
   229	
   230		// if camliType "permanode"
   231		Permanode *DescribedPermanode `json:"permanode,omitempty"`
   232	
   233		// if camliType "file"
   234		File *camtypes.FileInfo `json:"file,omitempty"`
   235		// if camliType "directory"
   236		Dir *camtypes.FileInfo `json:"dir,omitempty"`
   237		// if camliType "file", and File.IsImage()
   238		Image *camtypes.ImageInfo `json:"image,omitempty"`
   239		// if camliType "file" and media file
   240		MediaTags map[string]string `json:"mediaTags,omitempty"`
   241	
   242		// if camliType "directory"
   243		DirChildren []blob.Ref `json:"dirChildren,omitempty"`
   244	
   245		// Location specifies the location of the entity referenced
   246		// by the blob.
   247		//
   248		// If camliType is "file", then location comes from the metadata
   249		// (currently Exif) metadata of the file content.
   250		//
   251		// If camliType is "permanode", then location comes
   252		// from one of the following sources:
   253		//  1. Permanode attributes "latitude" and "longitude"
   254		//  2. Referenced permanode attributes (eg. for "foursquare.com:checkin"
   255		//     its "foursquareVenuePermanode")
   256		//  3. Location in permanode camliContent file metadata
   257		// The sources are checked in this order, the location from
   258		// the first source yielding a valid result is returned.
   259		Location *camtypes.Location `json:"location,omitempty"`
   260	
   261		// Stub is set if this is not loaded, but referenced.
   262		Stub bool `json:"-"`
   263	}
   264	
   265	func (m MetaMap) Get(br blob.Ref) *DescribedBlob {
   266		if !br.Valid() {
   267			return nil
   268		}
   269		return m[br.String()]
   270	}
   271	
   272	// URLSuffixPost returns the URL suffix for POST requests.
   273	func (dr *DescribeRequest) URLSuffixPost() string {
   274		return "camli/search/describe"
   275	}
   276	
   277	// URLSuffix returns the URL suffix for GET requests.
   278	// This is deprecated.
   279	func (dr *DescribeRequest) URLSuffix() string {
   280		var buf bytes.Buffer
   281		fmt.Fprintf(&buf, "camli/search/describe?depth=%d&maxdirchildren=%d",
   282			dr.depth(), dr.maxDirChildren())
   283		for _, br := range dr.BlobRefs {
   284			buf.WriteString("&blobref=")
   285			buf.WriteString(br.String())
   286		}
   287		if len(dr.BlobRefs) == 0 && dr.BlobRef.Valid() {
   288			buf.WriteString("&blobref=")
   289			buf.WriteString(dr.BlobRef.String())
   290		}
   291		if !dr.At.IsAnyZero() {
   292			buf.WriteString("&at=")
   293			buf.WriteString(dr.At.String())
   294		}
   295		return buf.String()
   296	}
   297	
   298	// fromHTTP panics with an httputil value on failure
   299	func (dr *DescribeRequest) fromHTTP(req *http.Request) {
   300		switch {
   301		case httputil.IsGet(req):
   302			dr.fromHTTPGet(req)
   303		case req.Method == "POST":
   304			dr.fromHTTPPost(req)
   305		default:
   306			panic("Unsupported method")
   307		}
   308	}
   309	
   310	func (dr *DescribeRequest) fromHTTPPost(req *http.Request) {
   311		err := json.NewDecoder(req.Body).Decode(dr)
   312		if err != nil {
   313			panic(err)
   314		}
   315	}
   316	
   317	func (dr *DescribeRequest) fromHTTPGet(req *http.Request) {
   318		req.ParseForm()
   319		if vv := req.Form["blobref"]; len(vv) > 1 {
   320			for _, brs := range vv {
   321				if br, ok := blob.Parse(brs); ok {
   322					dr.BlobRefs = append(dr.BlobRefs, br)
   323				} else {
   324					panic(httputil.InvalidParameterError("blobref"))
   325				}
   326			}
   327		} else {
   328			dr.BlobRef = httputil.MustGetBlobRef(req, "blobref")
   329		}
   330		dr.Depth = httputil.OptionalInt(req, "depth")
   331		dr.MaxDirChildren = httputil.OptionalInt(req, "maxdirchildren")
   332		dr.At = types.ParseTime3339OrZero(req.FormValue("at"))
   333	}
   334	
   335	// PermanodeFile returns in path the blobref of the described permanode
   336	// and the blobref of its File camliContent.
   337	// If b isn't a permanode, or doesn't have a camliContent that
   338	// is a file blob, ok is false.
   339	func (b *DescribedBlob) PermanodeFile() (path []blob.Ref, fi *camtypes.FileInfo, ok bool) {
   340		if b == nil || b.Permanode == nil {
   341			return
   342		}
   343		if contentRef := b.Permanode.Attr.Get("camliContent"); contentRef != "" {
   344			if cdes := b.Request.DescribedBlobStr(contentRef); cdes != nil && cdes.File != nil {
   345				return []blob.Ref{b.BlobRef, cdes.BlobRef}, cdes.File, true
   346			}
   347		}
   348		return
   349	}
   350	
   351	// PermanodeDir returns in path the blobref of the described permanode
   352	// and the blobref of its Directory camliContent.
   353	// If b isn't a permanode, or doesn't have a camliContent that
   354	// is a directory blob, ok is false.
   355	func (b *DescribedBlob) PermanodeDir() (path []blob.Ref, fi *camtypes.FileInfo, ok bool) {
   356		if b == nil || b.Permanode == nil {
   357			return
   358		}
   359		if contentRef := b.Permanode.Attr.Get("camliContent"); contentRef != "" {
   360			if cdes := b.Request.DescribedBlobStr(contentRef); cdes != nil && cdes.Dir != nil {
   361				return []blob.Ref{b.BlobRef, cdes.BlobRef}, cdes.Dir, true
   362			}
   363		}
   364		return
   365	}
   366	
   367	func (b *DescribedBlob) DomID() string {
   368		if b == nil {
   369			return ""
   370		}
   371		return b.BlobRef.DomID()
   372	}
   373	
   374	func (b *DescribedBlob) Title() string {
   375		if b == nil {
   376			return ""
   377		}
   378		if b.Permanode != nil {
   379			if t := b.Permanode.Attr.Get("title"); t != "" {
   380				return t
   381			}
   382			if contentRef := b.Permanode.Attr.Get("camliContent"); contentRef != "" {
   383				return b.Request.DescribedBlobStr(contentRef).Title()
   384			}
   385		}
   386		if b.File != nil {
   387			return b.File.FileName
   388		}
   389		if b.Dir != nil {
   390			return b.Dir.FileName
   391		}
   392		return ""
   393	}
   394	
   395	func (b *DescribedBlob) Description() string {
   396		if b == nil {
   397			return ""
   398		}
   399		if b.Permanode != nil {
   400			return b.Permanode.Attr.Get("description")
   401		}
   402		return ""
   403	}
   404	
   405	// Members returns all of b's children, as given by b's camliMember and camliPath:*
   406	// attributes. Only the first entry for a given camliPath attribute is used.
   407	func (b *DescribedBlob) Members() []*DescribedBlob {
   408		if b == nil {
   409			return nil
   410		}
   411		m := make([]*DescribedBlob, 0)
   412		if b.Permanode != nil {
   413			for _, bstr := range b.Permanode.Attr["camliMember"] {
   414				if br, ok := blob.Parse(bstr); ok {
   415					m = append(m, b.PeerBlob(br))
   416				}
   417			}
   418			for k, bstrs := range b.Permanode.Attr {
   419				if strings.HasPrefix(k, "camliPath:") && len(bstrs) > 0 {
   420					if br, ok := blob.Parse(bstrs[0]); ok {
   421						m = append(m, b.PeerBlob(br))
   422					}
   423				}
   424			}
   425		}
   426		return m
   427	}
   428	
   429	func (b *DescribedBlob) DirMembers() []*DescribedBlob {
   430		if b == nil || b.Dir == nil || len(b.DirChildren) == 0 {
   431			return nil
   432		}
   433	
   434		m := make([]*DescribedBlob, 0)
   435		for _, br := range b.DirChildren {
   436			m = append(m, b.PeerBlob(br))
   437		}
   438		return m
   439	}
   440	
   441	func (b *DescribedBlob) ContentRef() (br blob.Ref, ok bool) {
   442		if b != nil && b.Permanode != nil {
   443			if cref := b.Permanode.Attr.Get("camliContent"); cref != "" {
   444				return blob.Parse(cref)
   445			}
   446		}
   447		return
   448	}
   449	
   450	// DescribedBlobStr when given a blobref string returns a Description
   451	// or nil.  dr may be nil itself.
   452	func (dr *DescribeRequest) DescribedBlobStr(blobstr string) *DescribedBlob {
   453		if dr == nil {
   454			return nil
   455		}
   456		dr.mu.Lock()
   457		defer dr.mu.Unlock()
   458		return dr.m[blobstr]
   459	}
   460	
   461	// PeerBlob returns a DescribedBlob for the provided blobref.
   462	//
   463	// Unlike DescribedBlobStr, the returned DescribedBlob is never nil.
   464	//
   465	// If the blob was never loaded along with the the receiver (or if the
   466	// receiver is nil), a stub DescribedBlob is returned with its Stub
   467	// field set true.
   468	func (b *DescribedBlob) PeerBlob(br blob.Ref) *DescribedBlob {
   469		if b.Request == nil {
   470			return &DescribedBlob{BlobRef: br, Stub: true}
   471		}
   472		b.Request.mu.Lock()
   473		defer b.Request.mu.Unlock()
   474		return b.peerBlob(br)
   475	}
   476	
   477	// version of PeerBlob when b.Request.mu is already held.
   478	func (b *DescribedBlob) peerBlob(br blob.Ref) *DescribedBlob {
   479		if peer, ok := b.Request.m[br.String()]; ok {
   480			return peer
   481		}
   482		return &DescribedBlob{Request: b.Request, BlobRef: br, Stub: true}
   483	}
   484	
   485	type DescribedPermanode struct {
   486		Attr    url.Values `json:"attr"` // a map[string][]string
   487		ModTime time.Time  `json:"modtime,omitempty"`
   488	}
   489	
   490	// IsContainer returns whether the permanode has either named ("camliPath:"-prefixed) or unnamed
   491	// ("camliMember") member attributes.
   492	func (dp *DescribedPermanode) IsContainer() bool {
   493		if members := dp.Attr["camliMember"]; len(members) > 0 {
   494			return true
   495		}
   496		for k := range dp.Attr {
   497			if strings.HasPrefix(k, "camliPath:") {
   498				return true
   499			}
   500		}
   501		return false
   502	}
   503	
   504	// NewDescribeRequest returns a new DescribeRequest holding the state
   505	// of blobs and their summarized descriptions.  Use DescribeBlob
   506	// one or more times before calling Result.
   507	func (sh *Handler) NewDescribeRequest() *DescribeRequest {
   508		dr := new(DescribeRequest)
   509		sh.initDescribeRequest(dr)
   510		return dr
   511	}
   512	
   513	func (sh *Handler) initDescribeRequest(req *DescribeRequest) {
   514		if req.sh != nil {
   515			panic("already initialized")
   516		}
   517		req.sh = sh
   518		req.m = make(MetaMap)
   519		req.errs = make(map[string]error)
   520		req.wg = new(sync.WaitGroup)
   521	}
   522	
   523	type DescribeError map[string]error
   524	
   525	func (de DescribeError) Error() string {
   526		var buf bytes.Buffer
   527		for b, err := range de {
   528			fmt.Fprintf(&buf, "%s: %v; ", b, err)
   529		}
   530		return fmt.Sprintf("Errors (%d) describing blobs: %s", len(de), buf.String())
   531	}
   532	
   533	// Result waits for all outstanding lookups to complete and
   534	// returns the map of blobref (strings) to their described
   535	// results. The returned error is non-nil if any errors
   536	// occurred, and will be of type DescribeError.
   537	func (dr *DescribeRequest) Result() (desmap map[string]*DescribedBlob, err error) {
   538		dr.wg.Wait()
   539		// TODO: set "done" / locked flag, so no more DescribeBlob can
   540		// be called.
   541		if len(dr.errs) > 0 {
   542			return dr.m, DescribeError(dr.errs)
   543		}
   544		return dr.m, nil
   545	}
   546	
   547	func (dr *DescribeRequest) depth() int {
   548		if dr.Depth > 0 {
   549			return dr.Depth
   550		}
   551		return 1
   552	}
   553	
   554	func (dr *DescribeRequest) maxDirChildren() int {
   555		return sanitizeNumResults(dr.MaxDirChildren)
   556	}
   557	
   558	func (dr *DescribeRequest) metaMap() (map[string]*DescribedBlob, error) {
   559		dr.wg.Wait()
   560		dr.mu.Lock()
   561		defer dr.mu.Unlock()
   562		for k, err := range dr.errs {
   563			// TODO: include all?
   564			return nil, fmt.Errorf("error populating %s: %v", k, err)
   565		}
   566		m := make(map[string]*DescribedBlob)
   567		for k, desb := range dr.m {
   568			m[k] = desb
   569		}
   570		return m, nil
   571	}
   572	
   573	func (dr *DescribeRequest) describedBlob(b blob.Ref) *DescribedBlob {
   574		dr.mu.Lock()
   575		defer dr.mu.Unlock()
   576		bs := b.String()
   577		if des, ok := dr.m[bs]; ok {
   578			return des
   579		}
   580		des := &DescribedBlob{Request: dr, BlobRef: b}
   581		dr.m[bs] = des
   582		return des
   583	}
   584	
   585	func (dr *DescribeRequest) DescribeSync(ctx context.Context, br blob.Ref) (*DescribedBlob, error) {
   586		dr.StartDescribe(ctx, br, 1)
   587		res, err := dr.Result()
   588		if err != nil {
   589			return nil, err
   590		}
   591		return res[br.String()], nil
   592	}
   593	
   594	// StartDescribe starts a lookup of br, down to the provided depth.
   595	// It returns immediately. One should call Result to wait for the description to
   596	// be completed.
   597	func (dr *DescribeRequest) StartDescribe(ctx context.Context, br blob.Ref, depth int) {
   598		if depth <= 0 {
   599			return
   600		}
   601		dr.mu.Lock()
   602		defer dr.mu.Unlock()
   603		if dr.blobDesLock == nil {
   604			dr.blobDesLock = make(map[blob.Ref]*sync.Mutex)
   605		}
   606		desBlobMu, ok := dr.blobDesLock[br]
   607		if !ok {
   608			desBlobMu = new(sync.Mutex)
   609			dr.blobDesLock[br] = desBlobMu
   610		}
   611		if dr.started == nil {
   612			dr.started = make(map[blobrefAndDepth]bool)
   613		}
   614		key := blobrefAndDepth{br, depth}
   615		if dr.started[key] {
   616			return
   617		}
   618		dr.started[key] = true
   619		dr.wg.Add(1)
   620		go func() {
   621			defer dr.wg.Done()
   622			desBlobMu.Lock()
   623			defer desBlobMu.Unlock()
   624			dr.doDescribe(ctx, br, depth)
   625		}()
   626	}
   627	
   628	// requires dr.mu be held.
   629	func (r *DescribeRule) newMatches(br blob.Ref, dr *DescribeRequest) (brs []blob.Ref) {
   630		if r.IfResultRoot {
   631			if !dr.blobInitiallyRequested(br) {
   632				return nil
   633			}
   634		}
   635		if r.parentRule != nil {
   636			if _, ok := dr.resFromRule[r.parentRule][br]; !ok {
   637				return nil
   638			}
   639		}
   640		db, ok := dr.m[br.String()]
   641		if !ok || db.Permanode == nil {
   642			return nil
   643		}
   644		if t := r.IfCamliNodeType; t != "" {
   645			gotType := db.Permanode.Attr.Get("camliNodeType")
   646			if gotType != t {
   647				return nil
   648			}
   649		}
   650		for attr, vv := range db.Permanode.Attr {
   651			matches := false
   652			for _, matchAttr := range r.Attrs {
   653				if attr == matchAttr {
   654					matches = true
   655					break
   656				}
   657				if strings.HasSuffix(matchAttr, "*") && strings.HasPrefix(attr, strings.TrimSuffix(matchAttr, "*")) {
   658					matches = true
   659					break
   660				}
   661			}
   662			if !matches {
   663				continue
   664			}
   665			for _, v := range vv {
   666				if br, ok := blob.Parse(v); ok {
   667					brs = append(brs, br)
   668				}
   669			}
   670		}
   671		return brs
   672	}
   673	
   674	// dr.mu just be locked.
   675	func (dr *DescribeRequest) noteResultFromRule(rule *DescribeRule, br blob.Ref) {
   676		if dr.resFromRule == nil {
   677			dr.resFromRule = make(map[*DescribeRule]map[blob.Ref]bool)
   678		}
   679		m, ok := dr.resFromRule[rule]
   680		if !ok {
   681			m = make(map[blob.Ref]bool)
   682			dr.resFromRule[rule] = m
   683		}
   684		m[br] = true
   685	}
   686	
   687	func (dr *DescribeRequest) expandRules(ctx context.Context) error {
   688		loop := true
   689	
   690		for loop {
   691			loop = false
   692			dr.wg.Wait()
   693			dr.mu.Lock()
   694			len0 := len(dr.m)
   695			var new []blob.Ref
   696			for _, rule := range dr.flatRules() {
   697				dr.foreachResultBlob(func(br blob.Ref) {
   698					for _, nbr := range rule.newMatches(br, dr) {
   699						new = append(new, nbr)
   700						dr.noteResultFromRule(rule, nbr)
   701					}
   702				})
   703			}
   704			dr.mu.Unlock()
   705			for _, br := range new {
   706				dr.StartDescribe(ctx, br, 1)
   707			}
   708			dr.wg.Wait()
   709			dr.mu.Lock()
   710			len1 := len(dr.m)
   711			dr.mu.Unlock()
   712			loop = len0 != len1
   713		}
   714		return nil
   715	}
   716	
   717	func (dr *DescribeRequest) addError(br blob.Ref, err error) {
   718		if err == nil {
   719			return
   720		}
   721		dr.mu.Lock()
   722		defer dr.mu.Unlock()
   723		// TODO: append? meh.
   724		dr.errs[br.String()] = err
   725	}
   726	
   727	func (dr *DescribeRequest) doDescribe(ctx context.Context, br blob.Ref, depth int) {
   728		meta, err := dr.sh.index.GetBlobMeta(ctx, br)
   729		if err == os.ErrNotExist {
   730			return
   731		}
   732		if err != nil {
   733			dr.addError(br, err)
   734			return
   735		}
   736	
   737		// TODO: convert all this in terms of
   738		// DescribedBlob/DescribedPermanode/DescribedFile, not json
   739		// maps.  Then add JSON marhsallers to those types. Add tests.
   740		des := dr.describedBlob(br)
   741		if meta.CamliType != "" {
   742			des.setMIMEType("application/json; camliType=" + meta.CamliType)
   743		}
   744		des.Size = int64(meta.Size)
   745	
   746		switch des.CamliType {
   747		case "permanode":
   748			des.Permanode = new(DescribedPermanode)
   749			dr.populatePermanodeFields(ctx, des.Permanode, br, depth)
   750			var at time.Time
   751			if !dr.At.IsAnyZero() {
   752				at = dr.At.Time()
   753			}
   754			if loc, err := dr.sh.lh.PermanodeLocation(ctx, br, at, dr.sh.owner); err == nil {
   755				des.Location = &loc
   756			} else {
   757				if err != os.ErrNotExist {
   758					log.Printf("PermanodeLocation(permanode %s): %v", br, err)
   759				}
   760			}
   761		case "file":
   762			fi, err := dr.sh.index.GetFileInfo(ctx, br)
   763			if err != nil {
   764				if os.IsNotExist(err) {
   765					log.Printf("index.GetFileInfo(file %s) failed; index stale?", br)
   766				} else {
   767					dr.addError(br, err)
   768				}
   769				return
   770			}
   771			des.File = &fi
   772			if des.File.IsImage() {
   773				imgInfo, err := dr.sh.index.GetImageInfo(ctx, br)
   774				if err != nil {
   775					if !os.IsNotExist(err) {
   776						dr.addError(br, err)
   777					}
   778				} else {
   779					des.Image = &imgInfo
   780				}
   781			}
   782			if mediaTags, err := dr.sh.index.GetMediaTags(ctx, br); err == nil {
   783				des.MediaTags = mediaTags
   784			}
   785			if loc, err := dr.sh.index.GetFileLocation(ctx, br); err == nil {
   786				des.Location = &loc
   787			} else {
   788				if err != os.ErrNotExist {
   789					log.Printf("index.GetFileLocation(file %s): %v", br, err)
   790				}
   791			}
   792		case "directory":
   793			var g syncutil.Group
   794			g.Go(func() (err error) {
   795				fi, err := dr.sh.index.GetFileInfo(ctx, br)
   796				if os.IsNotExist(err) {
   797					log.Printf("index.GetFileInfo(directory %s) failed; index stale?", br)
   798				}
   799				if err == nil {
   800					des.Dir = &fi
   801				}
   802				return
   803			})
   804			g.Go(func() (err error) {
   805				des.DirChildren, err = dr.getDirMembers(ctx, br, depth)
   806				return
   807			})
   808			if err := g.Err(); err != nil {
   809				dr.addError(br, err)
   810			}
   811		}
   812	}
   813	
   814	func (dr *DescribeRequest) populatePermanodeFields(ctx context.Context, pi *DescribedPermanode, pn blob.Ref, depth int) {
   815		pi.Attr = make(url.Values)
   816		attr := pi.Attr
   817	
   818		claims, err := dr.sh.index.AppendClaims(ctx, nil, pn, dr.sh.owner.KeyID(), "")
   819		if err != nil {
   820			log.Printf("Error getting claims of %s: %v", pn.String(), err)
   821			dr.addError(pn, fmt.Errorf("Error getting claims of %s: %v", pn.String(), err))
   822			return
   823		}
   824	
   825		sort.Sort(camtypes.ClaimsByDate(claims))
   826	claimLoop:
   827		for _, cl := range claims {
   828			if !dr.At.IsAnyZero() {
   829				if cl.Date.After(dr.At.Time()) {
   830					continue
   831				}
   832			}
   833			switch cl.Type {
   834			default:
   835				continue
   836			case "del-attribute":
   837				if cl.Value == "" {
   838					delete(attr, cl.Attr)
   839				} else {
   840					sl := attr[cl.Attr]
   841					filtered := make([]string, 0, len(sl))
   842					for _, val := range sl {
   843						if val != cl.Value {
   844							filtered = append(filtered, val)
   845						}
   846					}
   847					attr[cl.Attr] = filtered
   848				}
   849			case "set-attribute":
   850				delete(attr, cl.Attr)
   851				fallthrough
   852			case "add-attribute":
   853				if cl.Value == "" {
   854					continue
   855				}
   856				sl, ok := attr[cl.Attr]
   857				if ok {
   858					for _, exist := range sl {
   859						if exist == cl.Value {
   860							continue claimLoop
   861						}
   862					}
   863				} else {
   864					sl = make([]string, 0, 1)
   865					attr[cl.Attr] = sl
   866				}
   867				attr[cl.Attr] = append(sl, cl.Value)
   868			}
   869			pi.ModTime = cl.Date
   870		}
   871	
   872		// Descend into any references in current attributes.
   873		for key, vals := range attr {
   874			dr.describeRefs(ctx, key, depth)
   875			for _, v := range vals {
   876				dr.describeRefs(ctx, v, depth)
   877			}
   878		}
   879	}
   880	
   881	func (dr *DescribeRequest) getDirMembers(ctx context.Context, br blob.Ref, depth int) ([]blob.Ref, error) {
   882		limit := dr.maxDirChildren()
   883		ch := make(chan blob.Ref)
   884		errch := make(chan error)
   885		go func() {
   886			errch <- dr.sh.index.GetDirMembers(ctx, br, ch, limit)
   887		}()
   888	
   889		var members []blob.Ref
   890		for child := range ch {
   891			dr.StartDescribe(ctx, child, depth)
   892			members = append(members, child)
   893		}
   894		if err := <-errch; err != nil {
   895			return nil, err
   896		}
   897		return members, nil
   898	}
   899	
   900	func (dr *DescribeRequest) describeRefs(ctx context.Context, str string, depth int) {
   901		for _, match := range blobRefPattern.FindAllString(str, -1) {
   902			if ref, ok := blob.ParseKnown(match); ok {
   903				dr.StartDescribe(ctx, ref, depth-1)
   904			}
   905		}
   906	}
   907	
   908	func (b *DescribedBlob) setMIMEType(mime string) {
   909		if strings.HasPrefix(mime, camliTypePrefix) {
   910			b.CamliType = strings.TrimPrefix(mime, camliTypePrefix)
   911		}
   912	}
Website layout inspired by memcached.
Content by the authors.