Home Download Docs Code Community
     1	/*
     2	Copyright 2011 The Perkeep Authors
     3	
     4	Licensed under the Apache License, Version 2.0 (the "License");
     5	you may not use this file except in compliance with the License.
     6	You may obtain a copy of the License at
     7	
     8	     http://www.apache.org/licenses/LICENSE-2.0
     9	
    10	Unless required by applicable law or agreed to in writing, software
    11	distributed under the License is distributed on an "AS IS" BASIS,
    12	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13	See the License for the specific language governing permissions and
    14	limitations under the License.
    15	*/
    16	
    17	package search
    18	
    19	import (
    20		"bytes"
    21		"context"
    22		"encoding/json"
    23		"errors"
    24		"fmt"
    25		"io/ioutil"
    26		"log"
    27		"net/http"
    28		"net/url"
    29		"regexp"
    30		"sort"
    31		"strconv"
    32		"strings"
    33		"time"
    34	
    35		"go4.org/jsonconfig"
    36		"go4.org/types"
    37		"perkeep.org/internal/httputil"
    38		"perkeep.org/pkg/blob"
    39		"perkeep.org/pkg/blobserver"
    40		"perkeep.org/pkg/index"
    41		"perkeep.org/pkg/jsonsign"
    42		"perkeep.org/pkg/schema"
    43		"perkeep.org/pkg/types/camtypes"
    44		"perkeep.org/pkg/types/serverconfig"
    45	)
    46	
    47	const buffered = 32     // arbitrary channel buffer size
    48	const maxResults = 1000 // arbitrary limit on the number of search results returned
    49	const defaultNumResults = 50
    50	
    51	// MaxImageSize is the maximum width or height in pixels that we will serve image
    52	// thumbnails at. It is used in the search result UI.
    53	const MaxImageSize = 2000
    54	
    55	var blobRefPattern = regexp.MustCompile(blob.Pattern)
    56	
    57	func init() {
    58		blobserver.RegisterHandlerConstructor("search", newHandlerFromConfig)
    59	}
    60	
    61	var (
    62		_ QueryDescriber = (*Handler)(nil)
    63	)
    64	
    65	// Handler handles search queries.
    66	type Handler struct {
    67		index index.Interface
    68		owner *index.Owner
    69		// optional for search aliases
    70		fetcher blob.Fetcher
    71	
    72		// Corpus optionally specifies the full in-memory metadata corpus
    73		// to use.
    74		// TODO: this may be required in the future, or folded into the index
    75		// interface.
    76		corpus *index.Corpus
    77	
    78		lh *index.LocationHelper
    79	
    80		// WebSocket hub
    81		wsHub *wsHub
    82	}
    83	
    84	// GetRecentPermanoder is the interface containing the GetRecentPermanodes method.
    85	type GetRecentPermanoder interface {
    86		// GetRecentPermanodes returns recently-modified permanodes.
    87		// This is a higher-level query returning more metadata than the index.GetRecentPermanodes,
    88		// which only scans the blobrefs but doesn't return anything about the permanodes.
    89		GetRecentPermanodes(context.Context, *RecentRequest) (*RecentResponse, error)
    90	}
    91	
    92	var _ GetRecentPermanoder = (*Handler)(nil)
    93	
    94	func NewHandler(ix index.Interface, owner *index.Owner) *Handler {
    95		sh := &Handler{
    96			index: ix,
    97			owner: owner,
    98		}
    99		sh.lh = index.NewLocationHelper(sh.index.(*index.Index))
   100		sh.wsHub = newWebsocketHub(sh)
   101		go sh.wsHub.run()
   102		sh.subscribeToNewBlobs()
   103		return sh
   104	}
   105	
   106	func (h *Handler) InitHandler(lh blobserver.FindHandlerByTyper) error {
   107		_, handler, err := lh.FindHandlerByType("storage-filesystem")
   108		if err != nil || handler == nil {
   109			return nil
   110		}
   111		h.fetcher = handler.(blob.Fetcher)
   112		registerKeyword(newNamedSearch(h))
   113		return nil
   114	}
   115	
   116	func (h *Handler) subscribeToNewBlobs() {
   117		ch := make(chan blob.Ref, buffered)
   118		blobserver.GetHub(h.index).RegisterListener(ch)
   119		go func() {
   120			ctx := context.Background()
   121			for br := range ch {
   122				h.index.RLock()
   123				bm, err := h.index.GetBlobMeta(ctx, br)
   124				if err == nil {
   125					h.wsHub.newBlobRecv <- bm.CamliType
   126				}
   127				h.index.RUnlock()
   128			}
   129		}()
   130	}
   131	
   132	func (h *Handler) SetCorpus(c *index.Corpus) {
   133		h.corpus = c
   134		h.lh.SetCorpus(c)
   135	}
   136	
   137	// SendStatusUpdate sends a JSON status map to any connected WebSocket clients.
   138	func (h *Handler) SendStatusUpdate(status json.RawMessage) {
   139		h.wsHub.statusUpdate <- status
   140	}
   141	
   142	func newHandlerFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (http.Handler, error) {
   143		indexPrefix := conf.RequiredString("index") // TODO: add optional help tips here?
   144		ownerCfg := conf.RequiredObject("owner")
   145		ownerId := ownerCfg.RequiredString("identity")
   146		ownerSecring := ownerCfg.RequiredString("secringFile")
   147	
   148		devBlockStartupPrefix := conf.OptionalString("devBlockStartupOn", "")
   149		slurpToMemory := conf.OptionalBool("slurpToMemory", false)
   150		if err := conf.Validate(); err != nil {
   151			return nil, err
   152		}
   153	
   154		if devBlockStartupPrefix != "" {
   155			_, err := ld.GetHandler(devBlockStartupPrefix)
   156			if err != nil {
   157				return nil, fmt.Errorf("search handler references bogus devBlockStartupOn handler %s: %v", devBlockStartupPrefix, err)
   158			}
   159		}
   160	
   161		indexHandler, err := ld.GetHandler(indexPrefix)
   162		if err != nil {
   163			return nil, fmt.Errorf("search config references unknown handler %q", indexPrefix)
   164		}
   165		indexer, ok := indexHandler.(index.Interface)
   166		if !ok {
   167			return nil, fmt.Errorf("search config references invalid indexer %q (actually a %T)", indexPrefix, indexHandler)
   168		}
   169	
   170		owner, err := newOwner(serverconfig.Owner{
   171			Identity:    ownerId,
   172			SecringFile: ownerSecring,
   173		})
   174		if err != nil {
   175			return nil, fmt.Errorf("could not create Owner %v", err)
   176		}
   177		h := NewHandler(indexer, owner)
   178	
   179		if slurpToMemory {
   180			ii := indexer.(*index.Index)
   181			ii.Lock()
   182			corpus, err := ii.KeepInMemory()
   183			if err != nil {
   184				ii.Unlock()
   185				return nil, fmt.Errorf("error slurping index to memory: %v", err)
   186			}
   187			h.SetCorpus(corpus)
   188			ii.Unlock()
   189		}
   190	
   191		return h, nil
   192	}
   193	
   194	func newOwner(ownerCfg serverconfig.Owner) (*index.Owner, error) {
   195		entity, err := jsonsign.EntityFromSecring(ownerCfg.Identity, ownerCfg.SecringFile)
   196		if err != nil {
   197			return nil, err
   198		}
   199		armoredPublicKey, err := jsonsign.ArmoredPublicKey(entity)
   200		if err != nil {
   201			return nil, err
   202		}
   203		return index.NewOwner(ownerCfg.Identity, blob.RefFromString(armoredPublicKey)), nil
   204	}
   205	
   206	// Owner returns Handler owner's public key blobref.
   207	// TODO(mpl): we're changing the index & search funcs to take a keyID (string)
   208	// or an *index.Owner, so any new func should probably not take/use h.Owner()
   209	// either.
   210	func (h *Handler) Owner() blob.Ref {
   211		// TODO: figure out a plan for an owner having multiple active public keys, or public
   212		// key rotation
   213		return h.owner.BlobRef()
   214	}
   215	
   216	func (h *Handler) Index() index.Interface {
   217		return h.index
   218	}
   219	
   220	// HasLegacySHA1 reports whether the server has legacy SHA-1 blobs indexed.
   221	func (h *Handler) HasLegacySHA1() bool {
   222		idx, ok := h.index.(*index.Index)
   223		if !ok {
   224			log.Printf("Cannot guess for legacy SHA1 because we don't have an *index.Index")
   225			return false
   226		}
   227		ok, err := idx.HasLegacySHA1()
   228		if err != nil {
   229			log.Printf("Cannot guess for legacy SHA1: %v", err)
   230			return false
   231		}
   232		return ok
   233	}
   234	
   235	var getHandler = map[string]func(*Handler, http.ResponseWriter, *http.Request){
   236		"ws":              (*Handler).serveWebSocket,
   237		"recent":          (*Handler).serveRecentPermanodes,
   238		"permanodeattr":   (*Handler).servePermanodesWithAttr,
   239		"describe":        (*Handler).serveDescribe,
   240		"claims":          (*Handler).serveClaims,
   241		"files":           (*Handler).serveFiles,
   242		"signerattrvalue": (*Handler).serveSignerAttrValue,
   243		"signerpaths":     (*Handler).serveSignerPaths,
   244		"edgesto":         (*Handler).serveEdgesTo,
   245	}
   246	
   247	var postHandler = map[string]func(*Handler, http.ResponseWriter, *http.Request){
   248		"describe": (*Handler).serveDescribe,
   249		"query":    (*Handler).serveQuery,
   250	}
   251	
   252	func (h *Handler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
   253		suffix := httputil.PathSuffix(req)
   254	
   255		handlers := getHandler
   256		switch {
   257		case httputil.IsGet(req):
   258			// use default from above
   259		case req.Method == "POST":
   260			handlers = postHandler
   261		default:
   262			handlers = nil
   263		}
   264		fn := handlers[strings.TrimPrefix(suffix, "camli/search/")]
   265		if fn != nil {
   266			fn(h, rw, req)
   267			return
   268		}
   269	
   270		// TODO: discovery for the endpoints & better error message with link to discovery info
   271		ret := camtypes.SearchErrorResponse{
   272			Error:     "Unsupported search path or method",
   273			ErrorType: "input",
   274		}
   275		httputil.ReturnJSON(rw, &ret)
   276	}
   277	
   278	// sanitizeNumResults takes n as a requested number of search results and sanitizes it.
   279	func sanitizeNumResults(n int) int {
   280		if n <= 0 || n > maxResults {
   281			return defaultNumResults
   282		}
   283		return n
   284	}
   285	
   286	// RecentRequest is a request to get a RecentResponse.
   287	type RecentRequest struct {
   288		N      int       // if zero, default number of results
   289		Before time.Time // if zero, now
   290	}
   291	
   292	func (r *RecentRequest) URLSuffix() string {
   293		var buf bytes.Buffer
   294		fmt.Fprintf(&buf, "camli/search/recent?n=%d", r.n())
   295		if !r.Before.IsZero() {
   296			fmt.Fprintf(&buf, "&before=%s", types.Time3339(r.Before))
   297		}
   298		return buf.String()
   299	}
   300	
   301	// fromHTTP panics with an httputil value on failure
   302	func (r *RecentRequest) fromHTTP(req *http.Request) {
   303		r.N, _ = strconv.Atoi(req.FormValue("n"))
   304		if before := req.FormValue("before"); before != "" {
   305			r.Before = time.Time(types.ParseTime3339OrZero(before))
   306		}
   307	}
   308	
   309	// n returns the sanitized maximum number of search results.
   310	func (r *RecentRequest) n() int {
   311		return sanitizeNumResults(r.N)
   312	}
   313	
   314	// WithAttrRequest is a request to get a WithAttrResponse.
   315	type WithAttrRequest struct {
   316		N      int      // max number of results
   317		Signer blob.Ref // if nil, will use the server's default owner (if configured)
   318		// Requested attribute. If blank, all attributes are searched (for Value)
   319		// as fulltext.
   320		Attr string
   321		// Value of the requested attribute. If blank, permanodes which have
   322		// request.Attr as an attribute are searched.
   323		Value string
   324		Fuzzy bool // fulltext search (if supported).
   325		// At, if non-zero, specifies that the attribute must have been set at
   326		// the latest at At.
   327		At time.Time
   328	}
   329	
   330	func (r *WithAttrRequest) URLSuffix() string {
   331		s := fmt.Sprintf("camli/search/permanodeattr?signer=%v&value=%v&fuzzy=%v&attr=%v&max=%v",
   332			r.Signer, url.QueryEscape(r.Value), r.Fuzzy, r.Attr, r.N)
   333		if !r.At.IsZero() {
   334			s += fmt.Sprintf("&at=%s", types.Time3339(r.At))
   335		}
   336		return s
   337	}
   338	
   339	// fromHTTP panics with an httputil value on failure
   340	func (r *WithAttrRequest) fromHTTP(req *http.Request) {
   341		r.Signer = blob.ParseOrZero(req.FormValue("signer"))
   342		r.Value = req.FormValue("value")
   343		fuzzy := req.FormValue("fuzzy") // exact match if empty
   344		fuzzyMatch := false
   345		if fuzzy != "" {
   346			lowered := strings.ToLower(fuzzy)
   347			if lowered == "true" || lowered == "t" {
   348				fuzzyMatch = true
   349			}
   350		}
   351		r.Attr = req.FormValue("attr") // all attributes if empty
   352		if r.Attr == "" {              // and force fuzzy in that case.
   353			fuzzyMatch = true
   354		}
   355		r.Fuzzy = fuzzyMatch
   356		max := req.FormValue("max")
   357		if max != "" {
   358			maxR, err := strconv.Atoi(max)
   359			if err != nil {
   360				panic(httputil.InvalidParameterError("max"))
   361			}
   362			r.N = maxR
   363		}
   364		r.N = r.n()
   365		if at := req.FormValue("at"); at != "" {
   366			r.At = time.Time(types.ParseTime3339OrZero(at))
   367		}
   368	}
   369	
   370	// n returns the sanitized maximum number of search results.
   371	func (r *WithAttrRequest) n() int {
   372		return sanitizeNumResults(r.N)
   373	}
   374	
   375	// ClaimsRequest is a request to get a ClaimsResponse.
   376	type ClaimsRequest struct {
   377		Permanode blob.Ref
   378	
   379		// AttrFilter optionally filters claims about the given attribute.
   380		// If empty, all claims for the given Permanode are returned.
   381		AttrFilter string
   382	}
   383	
   384	func (r *ClaimsRequest) URLSuffix() string {
   385		return fmt.Sprintf("camli/search/claims?permanode=%v&attrFilter=%s",
   386			r.Permanode, url.QueryEscape(r.AttrFilter))
   387	}
   388	
   389	// fromHTTP panics with an httputil value on failure
   390	func (r *ClaimsRequest) fromHTTP(req *http.Request) {
   391		r.Permanode = httputil.MustGetBlobRef(req, "permanode")
   392		r.AttrFilter = req.FormValue("attrFilter")
   393	}
   394	
   395	// SignerPathsRequest is a request to get a SignerPathsResponse.
   396	type SignerPathsRequest struct {
   397		Signer blob.Ref
   398		Target blob.Ref
   399	}
   400	
   401	// fromHTTP panics with an httputil value on failure
   402	func (r *SignerPathsRequest) fromHTTP(req *http.Request) {
   403		r.Signer = httputil.MustGetBlobRef(req, "signer")
   404		r.Target = httputil.MustGetBlobRef(req, "target")
   405	}
   406	
   407	// EdgesRequest is a request to get an EdgesResponse.
   408	type EdgesRequest struct {
   409		// The blob we want to find as a reference.
   410		ToRef blob.Ref
   411	}
   412	
   413	// fromHTTP panics with an httputil value on failure
   414	func (r *EdgesRequest) fromHTTP(req *http.Request) {
   415		r.ToRef = httputil.MustGetBlobRef(req, "blobref")
   416	}
   417	
   418	// TODO(mpl): it looks like we never populate RecentResponse.Error*, shouldn't we remove them?
   419	// Same for WithAttrResponse. I suppose it doesn't matter much if we end up removing GetRecentPermanodes anyway...
   420	
   421	// RecentResponse is the JSON response from $searchRoot/camli/search/recent.
   422	type RecentResponse struct {
   423		Recent []*RecentItem `json:"recent"`
   424		Meta   MetaMap       `json:"meta"`
   425	
   426		Error     string `json:"error,omitempty"`
   427		ErrorType string `json:"errorType,omitempty"`
   428	}
   429	
   430	func (r *RecentResponse) Err() error {
   431		if r.Error != "" || r.ErrorType != "" {
   432			if r.ErrorType != "" {
   433				return fmt.Errorf("%s: %s", r.ErrorType, r.Error)
   434			}
   435			return errors.New(r.Error)
   436		}
   437		return nil
   438	}
   439	
   440	// WithAttrResponse is the JSON response from $searchRoot/camli/search/permanodeattr.
   441	type WithAttrResponse struct {
   442		WithAttr []*WithAttrItem `json:"withAttr"`
   443		Meta     MetaMap         `json:"meta"`
   444	
   445		Error     string `json:"error,omitempty"`
   446		ErrorType string `json:"errorType,omitempty"`
   447	}
   448	
   449	func (r *WithAttrResponse) Err() error {
   450		if r.Error != "" || r.ErrorType != "" {
   451			if r.ErrorType != "" {
   452				return fmt.Errorf("%s: %s", r.ErrorType, r.Error)
   453			}
   454			return errors.New(r.Error)
   455		}
   456		return nil
   457	}
   458	
   459	// ClaimsResponse is the JSON response from $searchRoot/camli/search/claims.
   460	type ClaimsResponse struct {
   461		Claims []*ClaimsItem `json:"claims"`
   462	}
   463	
   464	// SignerPathsResponse is the JSON response from $searchRoot/camli/search/signerpaths.
   465	type SignerPathsResponse struct {
   466		Paths []*SignerPathsItem `json:"paths"`
   467		Meta  MetaMap            `json:"meta"`
   468	}
   469	
   470	// A RecentItem is an item returned from $searchRoot/camli/search/recent in the "recent" list.
   471	type RecentItem struct {
   472		BlobRef blob.Ref       `json:"blobref"`
   473		ModTime types.Time3339 `json:"modtime"`
   474		Owner   blob.Ref       `json:"owner"`
   475	}
   476	
   477	// A WithAttrItem is an item returned from $searchRoot/camli/search/permanodeattr.
   478	type WithAttrItem struct {
   479		Permanode blob.Ref `json:"permanode"`
   480	}
   481	
   482	// A ClaimsItem is an item returned from $searchRoot/camli/search/claims.
   483	type ClaimsItem struct {
   484		BlobRef   blob.Ref       `json:"blobref"`
   485		Signer    blob.Ref       `json:"signer"`
   486		Permanode blob.Ref       `json:"permanode"`
   487		Date      types.Time3339 `json:"date"`
   488		Type      string         `json:"type"`
   489		Attr      string         `json:"attr,omitempty"`
   490		Value     string         `json:"value,omitempty"`
   491	}
   492	
   493	// A SignerPathsItem is an item returned from $searchRoot/camli/search/signerpaths.
   494	type SignerPathsItem struct {
   495		ClaimRef blob.Ref `json:"claimRef"`
   496		BaseRef  blob.Ref `json:"baseRef"`
   497		Suffix   string   `json:"suffix"`
   498	}
   499	
   500	// EdgesResponse is the JSON response from $searchRoot/camli/search/edgesto.
   501	type EdgesResponse struct {
   502		ToRef   blob.Ref    `json:"toRef"`
   503		EdgesTo []*EdgeItem `json:"edgesTo"`
   504	}
   505	
   506	// An EdgeItem is an item returned from $searchRoot/camli/search/edgesto.
   507	type EdgeItem struct {
   508		From     blob.Ref         `json:"from"`
   509		FromType schema.CamliType `json:"fromType"`
   510	}
   511	
   512	var testHookBug121 = func() {}
   513	
   514	// GetRecentPermanodes returns recently-modified permanodes.
   515	func (h *Handler) GetRecentPermanodes(ctx context.Context, req *RecentRequest) (*RecentResponse, error) {
   516		h.index.RLock()
   517		defer h.index.RUnlock()
   518	
   519		ch := make(chan camtypes.RecentPermanode)
   520		errch := make(chan error, 1)
   521		before := time.Now()
   522		if !req.Before.IsZero() {
   523			before = req.Before
   524		}
   525		go func() {
   526			// TODO(mpl): change index funcs to take signer keyID. dont care for now, just
   527			// fixing the essential search and describe ones.
   528			errch <- h.index.GetRecentPermanodes(ctx, ch, h.owner.BlobRef(), req.n(), before)
   529		}()
   530	
   531		dr := h.NewDescribeRequest()
   532	
   533		var recent []*RecentItem
   534		for res := range ch {
   535			dr.StartDescribe(ctx, res.Permanode, 2)
   536			recent = append(recent, &RecentItem{
   537				BlobRef: res.Permanode,
   538				Owner:   res.Signer,
   539				ModTime: types.Time3339(res.LastModTime),
   540			})
   541			testHookBug121() // http://perkeep.org/issue/121
   542		}
   543	
   544		if err := <-errch; err != nil {
   545			return nil, err
   546		}
   547	
   548		metaMap, err := dr.metaMap()
   549		if err != nil {
   550			return nil, err
   551		}
   552	
   553		res := &RecentResponse{
   554			Recent: recent,
   555			Meta:   metaMap,
   556		}
   557		return res, nil
   558	}
   559	
   560	func (h *Handler) serveRecentPermanodes(rw http.ResponseWriter, req *http.Request) {
   561		defer httputil.RecoverJSON(rw, req)
   562		var rr RecentRequest
   563		rr.fromHTTP(req)
   564		res, err := h.GetRecentPermanodes(req.Context(), &rr)
   565		if err != nil {
   566			httputil.ServeJSONError(rw, err)
   567			return
   568		}
   569		httputil.ReturnJSON(rw, res)
   570	}
   571	
   572	// GetPermanodesWithAttr returns permanodes with attribute req.Attr
   573	// having the req.Value as a value.
   574	// See WithAttrRequest for more details about the query.
   575	func (h *Handler) GetPermanodesWithAttr(req *WithAttrRequest) (*WithAttrResponse, error) {
   576		ctx := context.TODO()
   577	
   578		h.index.RLock()
   579		defer h.index.RUnlock()
   580	
   581		ch := make(chan blob.Ref, buffered)
   582		errch := make(chan error, 1)
   583		go func() {
   584			signer := req.Signer
   585			if !signer.Valid() {
   586				signer = h.owner.BlobRef()
   587			}
   588			errch <- h.index.SearchPermanodesWithAttr(ctx, ch,
   589				&camtypes.PermanodeByAttrRequest{
   590					Attribute:  req.Attr,
   591					Query:      req.Value,
   592					Signer:     signer,
   593					FuzzyMatch: req.Fuzzy,
   594					MaxResults: req.N,
   595					At:         req.At,
   596				})
   597		}()
   598	
   599		dr := h.NewDescribeRequest()
   600	
   601		var withAttr []*WithAttrItem
   602		for res := range ch {
   603			dr.StartDescribe(ctx, res, 2)
   604			withAttr = append(withAttr, &WithAttrItem{
   605				Permanode: res,
   606			})
   607		}
   608	
   609		metaMap, err := dr.metaMap()
   610		if err != nil {
   611			return nil, err
   612		}
   613	
   614		if err := <-errch; err != nil {
   615			return nil, err
   616		}
   617	
   618		res := &WithAttrResponse{
   619			WithAttr: withAttr,
   620			Meta:     metaMap,
   621		}
   622		return res, nil
   623	}
   624	
   625	// servePermanodesWithAttr uses the indexer to search for the permanodes matching
   626	// the request.
   627	// The valid values for the "attr" key in the request (i.e the only attributes
   628	// for a permanode which are actually indexed as such) are "tag" and "title".
   629	func (h *Handler) servePermanodesWithAttr(rw http.ResponseWriter, req *http.Request) {
   630		defer httputil.RecoverJSON(rw, req)
   631		var wr WithAttrRequest
   632		wr.fromHTTP(req)
   633		res, err := h.GetPermanodesWithAttr(&wr)
   634		if err != nil {
   635			httputil.ServeJSONError(rw, err)
   636			return
   637		}
   638		httputil.ReturnJSON(rw, res)
   639	}
   640	
   641	// GetClaims returns the claims on req.Permanode signed by h.owner.
   642	func (h *Handler) GetClaims(req *ClaimsRequest) (*ClaimsResponse, error) {
   643		if !req.Permanode.Valid() {
   644			return nil, errors.New("error getting claims: nil permanode")
   645		}
   646		h.index.RLock()
   647		defer h.index.RUnlock()
   648	
   649		ctx := context.TODO()
   650		var claims []camtypes.Claim
   651		claims, err := h.index.AppendClaims(ctx, claims, req.Permanode, h.owner.KeyID(), req.AttrFilter)
   652		if err != nil {
   653			return nil, fmt.Errorf("Error getting claims of %s: %v", req.Permanode.String(), err)
   654		}
   655		sort.Sort(camtypes.ClaimsByDate(claims))
   656		var jclaims []*ClaimsItem
   657		for _, claim := range claims {
   658			jclaim := &ClaimsItem{
   659				BlobRef:   claim.BlobRef,
   660				Signer:    claim.Signer,
   661				Permanode: claim.Permanode,
   662				Date:      types.Time3339(claim.Date),
   663				Type:      claim.Type,
   664				Attr:      claim.Attr,
   665				Value:     claim.Value,
   666			}
   667			jclaims = append(jclaims, jclaim)
   668		}
   669	
   670		res := &ClaimsResponse{
   671			Claims: jclaims,
   672		}
   673		return res, nil
   674	}
   675	
   676	func (h *Handler) serveClaims(rw http.ResponseWriter, req *http.Request) {
   677		defer httputil.RecoverJSON(rw, req)
   678	
   679		h.index.RLock()
   680		defer h.index.RUnlock()
   681	
   682		var cr ClaimsRequest
   683		cr.fromHTTP(req)
   684		res, err := h.GetClaims(&cr)
   685		if err != nil {
   686			httputil.ServeJSONError(rw, err)
   687			return
   688		}
   689		httputil.ReturnJSON(rw, res)
   690	}
   691	
   692	func (h *Handler) serveFiles(rw http.ResponseWriter, req *http.Request) {
   693		var ret camtypes.FileSearchResponse
   694		defer httputil.ReturnJSON(rw, &ret)
   695	
   696		h.index.RLock()
   697		defer h.index.RUnlock()
   698	
   699		if err := req.ParseForm(); err != nil {
   700			ret.Error = err.Error()
   701			ret.ErrorType = "input"
   702			return
   703		}
   704		values, ok := req.Form["wholedigest"]
   705		if !ok {
   706			ret.Error = "Missing 'wholedigest' param"
   707			ret.ErrorType = "input"
   708			return
   709		}
   710		var digests []blob.Ref
   711		for _, v := range values {
   712			br, ok := blob.Parse(v)
   713			if !ok {
   714				ret.Error = "Invalid 'wholedigest' param"
   715				ret.ErrorType = "input"
   716				return
   717			}
   718			digests = append(digests, br)
   719		}
   720	
   721		files, err := h.index.ExistingFileSchemas(digests...)
   722		if err != nil {
   723			ret.Error = err.Error()
   724			ret.ErrorType = "server"
   725		}
   726		// the ui code expects an object
   727		if files == nil {
   728			files = make(index.WholeRefToFile)
   729		}
   730		ret.Files = files
   731	}
   732	
   733	// SignerAttrValueResponse is the JSON response to $search/camli/search/signerattrvalue
   734	type SignerAttrValueResponse struct {
   735		Permanode blob.Ref `json:"permanode"`
   736		Meta      MetaMap  `json:"meta"`
   737	}
   738	
   739	func (h *Handler) serveSignerAttrValue(rw http.ResponseWriter, req *http.Request) {
   740		defer httputil.RecoverJSON(rw, req)
   741		ctx := context.TODO()
   742		signer := httputil.MustGetBlobRef(req, "signer")
   743		attr := httputil.MustGet(req, "attr")
   744		value := httputil.MustGet(req, "value")
   745	
   746		h.index.RLock()
   747		defer h.index.RUnlock()
   748	
   749		pn, err := h.index.PermanodeOfSignerAttrValue(ctx, signer, attr, value)
   750		if err != nil {
   751			httputil.ServeJSONError(rw, err)
   752			return
   753		}
   754	
   755		dr := h.NewDescribeRequest()
   756		dr.StartDescribe(ctx, pn, 2)
   757		metaMap, err := dr.metaMap()
   758		if err != nil {
   759			httputil.ServeJSONError(rw, err)
   760			return
   761		}
   762	
   763		httputil.ReturnJSON(rw, &SignerAttrValueResponse{
   764			Permanode: pn,
   765			Meta:      metaMap,
   766		})
   767	}
   768	
   769	// EdgesTo returns edges that reference req.RefTo.
   770	// It filters out since-deleted permanode edges.
   771	func (h *Handler) EdgesTo(req *EdgesRequest) (*EdgesResponse, error) {
   772		ctx := context.TODO()
   773		h.index.RLock()
   774		defer h.index.RUnlock()
   775	
   776		toRef := req.ToRef
   777		toRefStr := toRef.String()
   778		var edgeItems []*EdgeItem
   779	
   780		edges, err := h.index.EdgesTo(toRef, nil)
   781		if err != nil {
   782			panic(err)
   783		}
   784	
   785		type edgeOrError struct {
   786			edge *EdgeItem // or nil
   787			err  error
   788		}
   789		resc := make(chan edgeOrError)
   790		verify := func(edge *camtypes.Edge) {
   791			db, err := h.NewDescribeRequest().DescribeSync(ctx, edge.From)
   792			if err != nil {
   793				resc <- edgeOrError{err: err}
   794				return
   795			}
   796			found := false
   797			if db.Permanode != nil {
   798				for attr, vv := range db.Permanode.Attr {
   799					if index.IsBlobReferenceAttribute(attr) {
   800						for _, v := range vv {
   801							if v == toRefStr {
   802								found = true
   803							}
   804						}
   805					}
   806				}
   807			}
   808			var ei *EdgeItem
   809			if found {
   810				ei = &EdgeItem{
   811					From:     edge.From,
   812					FromType: schema.TypePermanode,
   813				}
   814			}
   815			resc <- edgeOrError{edge: ei}
   816		}
   817		verifying := 0
   818		for _, edge := range edges {
   819			if edge.FromType == schema.TypePermanode {
   820				verifying++
   821				go verify(edge)
   822				continue
   823			}
   824			ei := &EdgeItem{
   825				From:     edge.From,
   826				FromType: edge.FromType,
   827			}
   828			edgeItems = append(edgeItems, ei)
   829		}
   830		for i := 0; i < verifying; i++ {
   831			res := <-resc
   832			if res.err != nil {
   833				return nil, res.err
   834			}
   835			if res.edge != nil {
   836				edgeItems = append(edgeItems, res.edge)
   837			}
   838		}
   839	
   840		return &EdgesResponse{
   841			ToRef:   toRef,
   842			EdgesTo: edgeItems,
   843		}, nil
   844	}
   845	
   846	// Unlike the index interface's EdgesTo method, the "edgesto" Handler
   847	// here additionally filters out since-deleted permanode edges.
   848	func (h *Handler) serveEdgesTo(rw http.ResponseWriter, req *http.Request) {
   849		defer httputil.RecoverJSON(rw, req)
   850		var er EdgesRequest
   851		er.fromHTTP(req)
   852		res, err := h.EdgesTo(&er)
   853		if err != nil {
   854			httputil.ServeJSONError(rw, err)
   855			return
   856		}
   857		httputil.ReturnJSON(rw, res)
   858	}
   859	
   860	func (h *Handler) serveQuery(rw http.ResponseWriter, req *http.Request) {
   861		defer httputil.RecoverJSON(rw, req)
   862	
   863		var sq SearchQuery
   864		if err := sq.FromHTTP(req); err != nil {
   865			httputil.ServeJSONError(rw, err)
   866			return
   867		}
   868	
   869		sr, err := h.Query(req.Context(), &sq)
   870		if err != nil {
   871			httputil.ServeJSONError(rw, err)
   872			return
   873		}
   874	
   875		httputil.ReturnJSON(rw, sr)
   876	}
   877	
   878	// GetSignerPaths returns paths with a target of req.Target.
   879	func (h *Handler) GetSignerPaths(req *SignerPathsRequest) (*SignerPathsResponse, error) {
   880		ctx := context.TODO()
   881		if !req.Signer.Valid() {
   882			return nil, errors.New("error getting signer paths: nil signer")
   883		}
   884		if !req.Target.Valid() {
   885			return nil, errors.New("error getting signer paths: nil target")
   886		}
   887		h.index.RLock()
   888		defer h.index.RUnlock()
   889	
   890		paths, err := h.index.PathsOfSignerTarget(ctx, req.Signer, req.Target)
   891		if err != nil {
   892			return nil, fmt.Errorf("Error getting paths of %s: %v", req.Target.String(), err)
   893		}
   894		var jpaths []*SignerPathsItem
   895		for _, path := range paths {
   896			jpaths = append(jpaths, &SignerPathsItem{
   897				ClaimRef: path.Claim,
   898				BaseRef:  path.Base,
   899				Suffix:   path.Suffix,
   900			})
   901		}
   902	
   903		dr := h.NewDescribeRequest()
   904		for _, path := range paths {
   905			dr.StartDescribe(ctx, path.Base, 2)
   906		}
   907		metaMap, err := dr.metaMap()
   908		if err != nil {
   909			return nil, err
   910		}
   911	
   912		res := &SignerPathsResponse{
   913			Paths: jpaths,
   914			Meta:  metaMap,
   915		}
   916		return res, nil
   917	}
   918	
   919	func (h *Handler) serveSignerPaths(rw http.ResponseWriter, req *http.Request) {
   920		defer httputil.RecoverJSON(rw, req)
   921		var sr SignerPathsRequest
   922		sr.fromHTTP(req)
   923	
   924		res, err := h.GetSignerPaths(&sr)
   925		if err != nil {
   926			httputil.ServeJSONError(rw, err)
   927			return
   928		}
   929		httputil.ReturnJSON(rw, res)
   930	}
   931	
   932	// EvalSearchInput checks if its input is JSON. If so it returns a Constraint constructed from that JSON. Otherwise
   933	// it assumes the input to be a search expression. It parses the expression and returns the parsed Constraint.
   934	func evalSearchInput(in string) (*Constraint, error) {
   935		if len(in) == 0 {
   936			return nil, fmt.Errorf("empty expression")
   937		}
   938		if strings.HasPrefix(in, "{") && strings.HasSuffix(in, "}") {
   939			cs := new(Constraint)
   940			if err := json.NewDecoder(strings.NewReader(in)).Decode(&cs); err != nil {
   941				return nil, err
   942			}
   943			return cs, nil
   944		} else {
   945			sq, err := parseExpression(context.TODO(), in)
   946			if err != nil {
   947				return nil, err
   948			}
   949			return sq.Constraint.Logical.B, nil
   950		}
   951	}
   952	
   953	// getNamed displays the search expression or constraint json for the requested alias.
   954	func (sh *Handler) getNamed(ctx context.Context, name string) (string, error) {
   955		if sh.fetcher == nil {
   956			return "", fmt.Errorf("GetNamed functionality not available")
   957		}
   958		sr, err := sh.Query(ctx, NamedSearch(name))
   959		if err != nil {
   960			return "", err
   961		}
   962	
   963		if len(sr.Blobs) < 1 {
   964			return "", fmt.Errorf("No named search found for: %s", name)
   965		}
   966		permaRef := sr.Blobs[0].Blob
   967		substRefS := sr.Describe.Meta.Get(permaRef).Permanode.Attr.Get("camliContent")
   968		br, ok := blob.Parse(substRefS)
   969		if !ok {
   970			return "", fmt.Errorf("Invalid blob ref: %s", substRefS)
   971		}
   972	
   973		reader, _, err := sh.fetcher.Fetch(ctx, br)
   974		if err != nil {
   975			return "", err
   976		}
   977		result, err := ioutil.ReadAll(reader)
   978		if err != nil {
   979			return "", err
   980		}
   981		return string(result), nil
   982	}
   983	
   984	// NamedSearch returns a *SearchQuery to find the permanode of the search alias "name".
   985	func NamedSearch(name string) *SearchQuery {
   986		return &SearchQuery{
   987			Constraint: &Constraint{
   988				Permanode: &PermanodeConstraint{
   989					Attr:  "camliNamedSearch",
   990					Value: name,
   991				},
   992			},
   993			Describe: &DescribeRequest{},
   994		}
   995	}
   996	
   997	const camliTypePrefix = "application/json; camliType="
Website layout inspired by memcached.
Content by the authors.