Home Download Docs Code Community
     1	/*
     2	Copyright 2011 The Perkeep Authors
     3	
     4	Licensed under the Apache License, Version 2.0 (the "License");
     5	you may not use this file except in compliance with the License.
     6	You may obtain a copy of the License at
     7	
     8	     http://www.apache.org/licenses/LICENSE-2.0
     9	
    10	Unless required by applicable law or agreed to in writing, software
    11	distributed under the License is distributed on an "AS IS" BASIS,
    12	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13	See the License for the specific language governing permissions and
    14	limitations under the License.
    15	*/
    16	
    17	package search
    18	
    19	import (
    20		"bytes"
    21		"context"
    22		"encoding/json"
    23		"errors"
    24		"fmt"
    25		"io/ioutil"
    26		"log"
    27		"net/http"
    28		"net/url"
    29		"regexp"
    30		"sort"
    31		"strconv"
    32		"strings"
    33		"time"
    34	
    35		"go4.org/jsonconfig"
    36		"go4.org/types"
    37		"perkeep.org/internal/httputil"
    38		"perkeep.org/pkg/blob"
    39		"perkeep.org/pkg/blobserver"
    40		"perkeep.org/pkg/index"
    41		"perkeep.org/pkg/jsonsign"
    42		"perkeep.org/pkg/types/camtypes"
    43		"perkeep.org/pkg/types/serverconfig"
    44	)
    45	
    46	const buffered = 32     // arbitrary channel buffer size
    47	const maxResults = 1000 // arbitrary limit on the number of search results returned
    48	const defaultNumResults = 50
    49	
    50	// MaxImageSize is the maximum width or height in pixels that we will serve image
    51	// thumbnails at. It is used in the search result UI.
    52	const MaxImageSize = 2000
    53	
    54	var blobRefPattern = regexp.MustCompile(blob.Pattern)
    55	
    56	func init() {
    57		blobserver.RegisterHandlerConstructor("search", newHandlerFromConfig)
    58	}
    59	
    60	var (
    61		_ QueryDescriber = (*Handler)(nil)
    62	)
    63	
    64	// Handler handles search queries.
    65	type Handler struct {
    66		index index.Interface
    67		owner *index.Owner
    68		// optional for search aliases
    69		fetcher blob.Fetcher
    70	
    71		// Corpus optionally specifies the full in-memory metadata corpus
    72		// to use.
    73		// TODO: this may be required in the future, or folded into the index
    74		// interface.
    75		corpus *index.Corpus
    76	
    77		lh *index.LocationHelper
    78	
    79		// WebSocket hub
    80		wsHub *wsHub
    81	}
    82	
    83	// GetRecentPermanoder is the interface containing the GetRecentPermanodes method.
    84	type GetRecentPermanoder interface {
    85		// GetRecentPermanodes returns recently-modified permanodes.
    86		// This is a higher-level query returning more metadata than the index.GetRecentPermanodes,
    87		// which only scans the blobrefs but doesn't return anything about the permanodes.
    88		GetRecentPermanodes(context.Context, *RecentRequest) (*RecentResponse, error)
    89	}
    90	
    91	var _ GetRecentPermanoder = (*Handler)(nil)
    92	
    93	func NewHandler(ix index.Interface, owner *index.Owner) *Handler {
    94		sh := &Handler{
    95			index: ix,
    96			owner: owner,
    97		}
    98		sh.lh = index.NewLocationHelper(sh.index.(*index.Index))
    99		sh.wsHub = newWebsocketHub(sh)
   100		go sh.wsHub.run()
   101		sh.subscribeToNewBlobs()
   102		return sh
   103	}
   104	
   105	func (h *Handler) InitHandler(lh blobserver.FindHandlerByTyper) error {
   106		_, handler, err := lh.FindHandlerByType("storage-filesystem")
   107		if err != nil || handler == nil {
   108			return nil
   109		}
   110		h.fetcher = handler.(blob.Fetcher)
   111		registerKeyword(newNamedSearch(h))
   112		return nil
   113	}
   114	
   115	func (h *Handler) subscribeToNewBlobs() {
   116		ch := make(chan blob.Ref, buffered)
   117		blobserver.GetHub(h.index).RegisterListener(ch)
   118		go func() {
   119			ctx := context.Background()
   120			for br := range ch {
   121				h.index.RLock()
   122				bm, err := h.index.GetBlobMeta(ctx, br)
   123				if err == nil {
   124					h.wsHub.newBlobRecv <- bm.CamliType
   125				}
   126				h.index.RUnlock()
   127			}
   128		}()
   129	}
   130	
   131	func (h *Handler) SetCorpus(c *index.Corpus) {
   132		h.corpus = c
   133		h.lh.SetCorpus(c)
   134	}
   135	
   136	// SendStatusUpdate sends a JSON status map to any connected WebSocket clients.
   137	func (h *Handler) SendStatusUpdate(status json.RawMessage) {
   138		h.wsHub.statusUpdate <- status
   139	}
   140	
   141	func newHandlerFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (http.Handler, error) {
   142		indexPrefix := conf.RequiredString("index") // TODO: add optional help tips here?
   143		ownerCfg := conf.RequiredObject("owner")
   144		ownerId := ownerCfg.RequiredString("identity")
   145		ownerSecring := ownerCfg.RequiredString("secringFile")
   146	
   147		devBlockStartupPrefix := conf.OptionalString("devBlockStartupOn", "")
   148		slurpToMemory := conf.OptionalBool("slurpToMemory", false)
   149		if err := conf.Validate(); err != nil {
   150			return nil, err
   151		}
   152	
   153		if devBlockStartupPrefix != "" {
   154			_, err := ld.GetHandler(devBlockStartupPrefix)
   155			if err != nil {
   156				return nil, fmt.Errorf("search handler references bogus devBlockStartupOn handler %s: %v", devBlockStartupPrefix, err)
   157			}
   158		}
   159	
   160		indexHandler, err := ld.GetHandler(indexPrefix)
   161		if err != nil {
   162			return nil, fmt.Errorf("search config references unknown handler %q", indexPrefix)
   163		}
   164		indexer, ok := indexHandler.(index.Interface)
   165		if !ok {
   166			return nil, fmt.Errorf("search config references invalid indexer %q (actually a %T)", indexPrefix, indexHandler)
   167		}
   168	
   169		owner, err := newOwner(serverconfig.Owner{
   170			Identity:    ownerId,
   171			SecringFile: ownerSecring,
   172		})
   173		if err != nil {
   174			return nil, fmt.Errorf("could not create Owner %v", err)
   175		}
   176		h := NewHandler(indexer, owner)
   177	
   178		if slurpToMemory {
   179			ii := indexer.(*index.Index)
   180			ii.Lock()
   181			corpus, err := ii.KeepInMemory()
   182			if err != nil {
   183				ii.Unlock()
   184				return nil, fmt.Errorf("error slurping index to memory: %v", err)
   185			}
   186			h.SetCorpus(corpus)
   187			ii.Unlock()
   188		}
   189	
   190		return h, nil
   191	}
   192	
   193	func newOwner(ownerCfg serverconfig.Owner) (*index.Owner, error) {
   194		entity, err := jsonsign.EntityFromSecring(ownerCfg.Identity, ownerCfg.SecringFile)
   195		if err != nil {
   196			return nil, err
   197		}
   198		armoredPublicKey, err := jsonsign.ArmoredPublicKey(entity)
   199		if err != nil {
   200			return nil, err
   201		}
   202		return index.NewOwner(ownerCfg.Identity, blob.RefFromString(armoredPublicKey)), nil
   203	}
   204	
   205	// Owner returns Handler owner's public key blobref.
   206	// TODO(mpl): we're changing the index & search funcs to take a keyID (string)
   207	// or an *index.Owner, so any new func should probably not take/use h.Owner()
   208	// either.
   209	func (h *Handler) Owner() blob.Ref {
   210		// TODO: figure out a plan for an owner having multiple active public keys, or public
   211		// key rotation
   212		return h.owner.BlobRef()
   213	}
   214	
   215	func (h *Handler) Index() index.Interface {
   216		return h.index
   217	}
   218	
   219	// HasLegacySHA1 reports whether the server has legacy SHA-1 blobs indexed.
   220	func (h *Handler) HasLegacySHA1() bool {
   221		idx, ok := h.index.(*index.Index)
   222		if !ok {
   223			log.Printf("Cannot guess for legacy SHA1 because we don't have an *index.Index")
   224			return false
   225		}
   226		ok, err := idx.HasLegacySHA1()
   227		if err != nil {
   228			log.Printf("Cannot guess for legacy SHA1: %v", err)
   229			return false
   230		}
   231		return ok
   232	}
   233	
   234	var getHandler = map[string]func(*Handler, http.ResponseWriter, *http.Request){
   235		"ws":              (*Handler).serveWebSocket,
   236		"recent":          (*Handler).serveRecentPermanodes,
   237		"permanodeattr":   (*Handler).servePermanodesWithAttr,
   238		"describe":        (*Handler).serveDescribe,
   239		"claims":          (*Handler).serveClaims,
   240		"files":           (*Handler).serveFiles,
   241		"signerattrvalue": (*Handler).serveSignerAttrValue,
   242		"signerpaths":     (*Handler).serveSignerPaths,
   243		"edgesto":         (*Handler).serveEdgesTo,
   244	}
   245	
   246	var postHandler = map[string]func(*Handler, http.ResponseWriter, *http.Request){
   247		"describe": (*Handler).serveDescribe,
   248		"query":    (*Handler).serveQuery,
   249	}
   250	
   251	func (h *Handler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
   252		suffix := httputil.PathSuffix(req)
   253	
   254		handlers := getHandler
   255		switch {
   256		case httputil.IsGet(req):
   257			// use default from above
   258		case req.Method == "POST":
   259			handlers = postHandler
   260		default:
   261			handlers = nil
   262		}
   263		fn := handlers[strings.TrimPrefix(suffix, "camli/search/")]
   264		if fn != nil {
   265			fn(h, rw, req)
   266			return
   267		}
   268	
   269		// TODO: discovery for the endpoints & better error message with link to discovery info
   270		ret := camtypes.SearchErrorResponse{
   271			Error:     "Unsupported search path or method",
   272			ErrorType: "input",
   273		}
   274		httputil.ReturnJSON(rw, &ret)
   275	}
   276	
   277	// sanitizeNumResults takes n as a requested number of search results and sanitizes it.
   278	func sanitizeNumResults(n int) int {
   279		if n <= 0 || n > maxResults {
   280			return defaultNumResults
   281		}
   282		return n
   283	}
   284	
   285	// RecentRequest is a request to get a RecentResponse.
   286	type RecentRequest struct {
   287		N      int       // if zero, default number of results
   288		Before time.Time // if zero, now
   289	}
   290	
   291	func (r *RecentRequest) URLSuffix() string {
   292		var buf bytes.Buffer
   293		fmt.Fprintf(&buf, "camli/search/recent?n=%d", r.n())
   294		if !r.Before.IsZero() {
   295			fmt.Fprintf(&buf, "&before=%s", types.Time3339(r.Before))
   296		}
   297		return buf.String()
   298	}
   299	
   300	// fromHTTP panics with an httputil value on failure
   301	func (r *RecentRequest) fromHTTP(req *http.Request) {
   302		r.N, _ = strconv.Atoi(req.FormValue("n"))
   303		if before := req.FormValue("before"); before != "" {
   304			r.Before = time.Time(types.ParseTime3339OrZero(before))
   305		}
   306	}
   307	
   308	// n returns the sanitized maximum number of search results.
   309	func (r *RecentRequest) n() int {
   310		return sanitizeNumResults(r.N)
   311	}
   312	
   313	// WithAttrRequest is a request to get a WithAttrResponse.
   314	type WithAttrRequest struct {
   315		N      int      // max number of results
   316		Signer blob.Ref // if nil, will use the server's default owner (if configured)
   317		// Requested attribute. If blank, all attributes are searched (for Value)
   318		// as fulltext.
   319		Attr string
   320		// Value of the requested attribute. If blank, permanodes which have
   321		// request.Attr as an attribute are searched.
   322		Value string
   323		Fuzzy bool // fulltext search (if supported).
   324		// At, if non-zero, specifies that the attribute must have been set at
   325		// the latest at At.
   326		At time.Time
   327	}
   328	
   329	func (r *WithAttrRequest) URLSuffix() string {
   330		s := fmt.Sprintf("camli/search/permanodeattr?signer=%v&value=%v&fuzzy=%v&attr=%v&max=%v",
   331			r.Signer, url.QueryEscape(r.Value), r.Fuzzy, r.Attr, r.N)
   332		if !r.At.IsZero() {
   333			s += fmt.Sprintf("&at=%s", types.Time3339(r.At))
   334		}
   335		return s
   336	}
   337	
   338	// fromHTTP panics with an httputil value on failure
   339	func (r *WithAttrRequest) fromHTTP(req *http.Request) {
   340		r.Signer = blob.ParseOrZero(req.FormValue("signer"))
   341		r.Value = req.FormValue("value")
   342		fuzzy := req.FormValue("fuzzy") // exact match if empty
   343		fuzzyMatch := false
   344		if fuzzy != "" {
   345			lowered := strings.ToLower(fuzzy)
   346			if lowered == "true" || lowered == "t" {
   347				fuzzyMatch = true
   348			}
   349		}
   350		r.Attr = req.FormValue("attr") // all attributes if empty
   351		if r.Attr == "" {              // and force fuzzy in that case.
   352			fuzzyMatch = true
   353		}
   354		r.Fuzzy = fuzzyMatch
   355		max := req.FormValue("max")
   356		if max != "" {
   357			maxR, err := strconv.Atoi(max)
   358			if err != nil {
   359				panic(httputil.InvalidParameterError("max"))
   360			}
   361			r.N = maxR
   362		}
   363		r.N = r.n()
   364		if at := req.FormValue("at"); at != "" {
   365			r.At = time.Time(types.ParseTime3339OrZero(at))
   366		}
   367	}
   368	
   369	// n returns the sanitized maximum number of search results.
   370	func (r *WithAttrRequest) n() int {
   371		return sanitizeNumResults(r.N)
   372	}
   373	
   374	// ClaimsRequest is a request to get a ClaimsResponse.
   375	type ClaimsRequest struct {
   376		Permanode blob.Ref
   377	
   378		// AttrFilter optionally filters claims about the given attribute.
   379		// If empty, all claims for the given Permanode are returned.
   380		AttrFilter string
   381	}
   382	
   383	func (r *ClaimsRequest) URLSuffix() string {
   384		return fmt.Sprintf("camli/search/claims?permanode=%v&attrFilter=%s",
   385			r.Permanode, url.QueryEscape(r.AttrFilter))
   386	}
   387	
   388	// fromHTTP panics with an httputil value on failure
   389	func (r *ClaimsRequest) fromHTTP(req *http.Request) {
   390		r.Permanode = httputil.MustGetBlobRef(req, "permanode")
   391		r.AttrFilter = req.FormValue("attrFilter")
   392	}
   393	
   394	// SignerPathsRequest is a request to get a SignerPathsResponse.
   395	type SignerPathsRequest struct {
   396		Signer blob.Ref
   397		Target blob.Ref
   398	}
   399	
   400	// fromHTTP panics with an httputil value on failure
   401	func (r *SignerPathsRequest) fromHTTP(req *http.Request) {
   402		r.Signer = httputil.MustGetBlobRef(req, "signer")
   403		r.Target = httputil.MustGetBlobRef(req, "target")
   404	}
   405	
   406	// EdgesRequest is a request to get an EdgesResponse.
   407	type EdgesRequest struct {
   408		// The blob we want to find as a reference.
   409		ToRef blob.Ref
   410	}
   411	
   412	// fromHTTP panics with an httputil value on failure
   413	func (r *EdgesRequest) fromHTTP(req *http.Request) {
   414		r.ToRef = httputil.MustGetBlobRef(req, "blobref")
   415	}
   416	
   417	// TODO(mpl): it looks like we never populate RecentResponse.Error*, shouldn't we remove them?
   418	// Same for WithAttrResponse. I suppose it doesn't matter much if we end up removing GetRecentPermanodes anyway...
   419	
   420	// RecentResponse is the JSON response from $searchRoot/camli/search/recent.
   421	type RecentResponse struct {
   422		Recent []*RecentItem `json:"recent"`
   423		Meta   MetaMap       `json:"meta"`
   424	
   425		Error     string `json:"error,omitempty"`
   426		ErrorType string `json:"errorType,omitempty"`
   427	}
   428	
   429	func (r *RecentResponse) Err() error {
   430		if r.Error != "" || r.ErrorType != "" {
   431			if r.ErrorType != "" {
   432				return fmt.Errorf("%s: %s", r.ErrorType, r.Error)
   433			}
   434			return errors.New(r.Error)
   435		}
   436		return nil
   437	}
   438	
   439	// WithAttrResponse is the JSON response from $searchRoot/camli/search/permanodeattr.
   440	type WithAttrResponse struct {
   441		WithAttr []*WithAttrItem `json:"withAttr"`
   442		Meta     MetaMap         `json:"meta"`
   443	
   444		Error     string `json:"error,omitempty"`
   445		ErrorType string `json:"errorType,omitempty"`
   446	}
   447	
   448	func (r *WithAttrResponse) Err() error {
   449		if r.Error != "" || r.ErrorType != "" {
   450			if r.ErrorType != "" {
   451				return fmt.Errorf("%s: %s", r.ErrorType, r.Error)
   452			}
   453			return errors.New(r.Error)
   454		}
   455		return nil
   456	}
   457	
   458	// ClaimsResponse is the JSON response from $searchRoot/camli/search/claims.
   459	type ClaimsResponse struct {
   460		Claims []*ClaimsItem `json:"claims"`
   461	}
   462	
   463	// SignerPathsResponse is the JSON response from $searchRoot/camli/search/signerpaths.
   464	type SignerPathsResponse struct {
   465		Paths []*SignerPathsItem `json:"paths"`
   466		Meta  MetaMap            `json:"meta"`
   467	}
   468	
   469	// A RecentItem is an item returned from $searchRoot/camli/search/recent in the "recent" list.
   470	type RecentItem struct {
   471		BlobRef blob.Ref       `json:"blobref"`
   472		ModTime types.Time3339 `json:"modtime"`
   473		Owner   blob.Ref       `json:"owner"`
   474	}
   475	
   476	// A WithAttrItem is an item returned from $searchRoot/camli/search/permanodeattr.
   477	type WithAttrItem struct {
   478		Permanode blob.Ref `json:"permanode"`
   479	}
   480	
   481	// A ClaimsItem is an item returned from $searchRoot/camli/search/claims.
   482	type ClaimsItem struct {
   483		BlobRef   blob.Ref       `json:"blobref"`
   484		Signer    blob.Ref       `json:"signer"`
   485		Permanode blob.Ref       `json:"permanode"`
   486		Date      types.Time3339 `json:"date"`
   487		Type      string         `json:"type"`
   488		Attr      string         `json:"attr,omitempty"`
   489		Value     string         `json:"value,omitempty"`
   490	}
   491	
   492	// A SignerPathsItem is an item returned from $searchRoot/camli/search/signerpaths.
   493	type SignerPathsItem struct {
   494		ClaimRef blob.Ref `json:"claimRef"`
   495		BaseRef  blob.Ref `json:"baseRef"`
   496		Suffix   string   `json:"suffix"`
   497	}
   498	
   499	// EdgesResponse is the JSON response from $searchRoot/camli/search/edgesto.
   500	type EdgesResponse struct {
   501		ToRef   blob.Ref    `json:"toRef"`
   502		EdgesTo []*EdgeItem `json:"edgesTo"`
   503	}
   504	
   505	// An EdgeItem is an item returned from $searchRoot/camli/search/edgesto.
   506	type EdgeItem struct {
   507		From     blob.Ref `json:"from"`
   508		FromType string   `json:"fromType"`
   509	}
   510	
   511	var testHookBug121 = func() {}
   512	
   513	// GetRecentPermanodes returns recently-modified permanodes.
   514	func (h *Handler) GetRecentPermanodes(ctx context.Context, req *RecentRequest) (*RecentResponse, error) {
   515		h.index.RLock()
   516		defer h.index.RUnlock()
   517	
   518		ch := make(chan camtypes.RecentPermanode)
   519		errch := make(chan error, 1)
   520		before := time.Now()
   521		if !req.Before.IsZero() {
   522			before = req.Before
   523		}
   524		go func() {
   525			// TODO(mpl): change index funcs to take signer keyID. dont care for now, just
   526			// fixing the essential search and describe ones.
   527			errch <- h.index.GetRecentPermanodes(ctx, ch, h.owner.BlobRef(), req.n(), before)
   528		}()
   529	
   530		dr := h.NewDescribeRequest()
   531	
   532		var recent []*RecentItem
   533		for res := range ch {
   534			dr.StartDescribe(ctx, res.Permanode, 2)
   535			recent = append(recent, &RecentItem{
   536				BlobRef: res.Permanode,
   537				Owner:   res.Signer,
   538				ModTime: types.Time3339(res.LastModTime),
   539			})
   540			testHookBug121() // http://perkeep.org/issue/121
   541		}
   542	
   543		if err := <-errch; err != nil {
   544			return nil, err
   545		}
   546	
   547		metaMap, err := dr.metaMap()
   548		if err != nil {
   549			return nil, err
   550		}
   551	
   552		res := &RecentResponse{
   553			Recent: recent,
   554			Meta:   metaMap,
   555		}
   556		return res, nil
   557	}
   558	
   559	func (h *Handler) serveRecentPermanodes(rw http.ResponseWriter, req *http.Request) {
   560		defer httputil.RecoverJSON(rw, req)
   561		var rr RecentRequest
   562		rr.fromHTTP(req)
   563		res, err := h.GetRecentPermanodes(req.Context(), &rr)
   564		if err != nil {
   565			httputil.ServeJSONError(rw, err)
   566			return
   567		}
   568		httputil.ReturnJSON(rw, res)
   569	}
   570	
   571	// GetPermanodesWithAttr returns permanodes with attribute req.Attr
   572	// having the req.Value as a value.
   573	// See WithAttrRequest for more details about the query.
   574	func (h *Handler) GetPermanodesWithAttr(req *WithAttrRequest) (*WithAttrResponse, error) {
   575		ctx := context.TODO()
   576	
   577		h.index.RLock()
   578		defer h.index.RUnlock()
   579	
   580		ch := make(chan blob.Ref, buffered)
   581		errch := make(chan error, 1)
   582		go func() {
   583			signer := req.Signer
   584			if !signer.Valid() {
   585				signer = h.owner.BlobRef()
   586			}
   587			errch <- h.index.SearchPermanodesWithAttr(ctx, ch,
   588				&camtypes.PermanodeByAttrRequest{
   589					Attribute:  req.Attr,
   590					Query:      req.Value,
   591					Signer:     signer,
   592					FuzzyMatch: req.Fuzzy,
   593					MaxResults: req.N,
   594					At:         req.At,
   595				})
   596		}()
   597	
   598		dr := h.NewDescribeRequest()
   599	
   600		var withAttr []*WithAttrItem
   601		for res := range ch {
   602			dr.StartDescribe(ctx, res, 2)
   603			withAttr = append(withAttr, &WithAttrItem{
   604				Permanode: res,
   605			})
   606		}
   607	
   608		metaMap, err := dr.metaMap()
   609		if err != nil {
   610			return nil, err
   611		}
   612	
   613		if err := <-errch; err != nil {
   614			return nil, err
   615		}
   616	
   617		res := &WithAttrResponse{
   618			WithAttr: withAttr,
   619			Meta:     metaMap,
   620		}
   621		return res, nil
   622	}
   623	
   624	// servePermanodesWithAttr uses the indexer to search for the permanodes matching
   625	// the request.
   626	// The valid values for the "attr" key in the request (i.e the only attributes
   627	// for a permanode which are actually indexed as such) are "tag" and "title".
   628	func (h *Handler) servePermanodesWithAttr(rw http.ResponseWriter, req *http.Request) {
   629		defer httputil.RecoverJSON(rw, req)
   630		var wr WithAttrRequest
   631		wr.fromHTTP(req)
   632		res, err := h.GetPermanodesWithAttr(&wr)
   633		if err != nil {
   634			httputil.ServeJSONError(rw, err)
   635			return
   636		}
   637		httputil.ReturnJSON(rw, res)
   638	}
   639	
   640	// GetClaims returns the claims on req.Permanode signed by h.owner.
   641	func (h *Handler) GetClaims(req *ClaimsRequest) (*ClaimsResponse, error) {
   642		if !req.Permanode.Valid() {
   643			return nil, errors.New("error getting claims: nil permanode")
   644		}
   645		h.index.RLock()
   646		defer h.index.RUnlock()
   647	
   648		ctx := context.TODO()
   649		var claims []camtypes.Claim
   650		claims, err := h.index.AppendClaims(ctx, claims, req.Permanode, h.owner.KeyID(), req.AttrFilter)
   651		if err != nil {
   652			return nil, fmt.Errorf("Error getting claims of %s: %v", req.Permanode.String(), err)
   653		}
   654		sort.Sort(camtypes.ClaimsByDate(claims))
   655		var jclaims []*ClaimsItem
   656		for _, claim := range claims {
   657			jclaim := &ClaimsItem{
   658				BlobRef:   claim.BlobRef,
   659				Signer:    claim.Signer,
   660				Permanode: claim.Permanode,
   661				Date:      types.Time3339(claim.Date),
   662				Type:      claim.Type,
   663				Attr:      claim.Attr,
   664				Value:     claim.Value,
   665			}
   666			jclaims = append(jclaims, jclaim)
   667		}
   668	
   669		res := &ClaimsResponse{
   670			Claims: jclaims,
   671		}
   672		return res, nil
   673	}
   674	
   675	func (h *Handler) serveClaims(rw http.ResponseWriter, req *http.Request) {
   676		defer httputil.RecoverJSON(rw, req)
   677	
   678		h.index.RLock()
   679		defer h.index.RUnlock()
   680	
   681		var cr ClaimsRequest
   682		cr.fromHTTP(req)
   683		res, err := h.GetClaims(&cr)
   684		if err != nil {
   685			httputil.ServeJSONError(rw, err)
   686			return
   687		}
   688		httputil.ReturnJSON(rw, res)
   689	}
   690	
   691	func (h *Handler) serveFiles(rw http.ResponseWriter, req *http.Request) {
   692		var ret camtypes.FileSearchResponse
   693		defer httputil.ReturnJSON(rw, &ret)
   694	
   695		h.index.RLock()
   696		defer h.index.RUnlock()
   697	
   698		if err := req.ParseForm(); err != nil {
   699			ret.Error = err.Error()
   700			ret.ErrorType = "input"
   701			return
   702		}
   703		values, ok := req.Form["wholedigest"]
   704		if !ok {
   705			ret.Error = "Missing 'wholedigest' param"
   706			ret.ErrorType = "input"
   707			return
   708		}
   709		var digests []blob.Ref
   710		for _, v := range values {
   711			br, ok := blob.Parse(v)
   712			if !ok {
   713				ret.Error = "Invalid 'wholedigest' param"
   714				ret.ErrorType = "input"
   715				return
   716			}
   717			digests = append(digests, br)
   718		}
   719	
   720		files, err := h.index.ExistingFileSchemas(digests...)
   721		if err != nil {
   722			ret.Error = err.Error()
   723			ret.ErrorType = "server"
   724		}
   725		// the ui code expects an object
   726		if files == nil {
   727			files = make(index.WholeRefToFile)
   728		}
   729		ret.Files = files
   730	}
   731	
   732	// SignerAttrValueResponse is the JSON response to $search/camli/search/signerattrvalue
   733	type SignerAttrValueResponse struct {
   734		Permanode blob.Ref `json:"permanode"`
   735		Meta      MetaMap  `json:"meta"`
   736	}
   737	
   738	func (h *Handler) serveSignerAttrValue(rw http.ResponseWriter, req *http.Request) {
   739		defer httputil.RecoverJSON(rw, req)
   740		ctx := context.TODO()
   741		signer := httputil.MustGetBlobRef(req, "signer")
   742		attr := httputil.MustGet(req, "attr")
   743		value := httputil.MustGet(req, "value")
   744	
   745		h.index.RLock()
   746		defer h.index.RUnlock()
   747	
   748		pn, err := h.index.PermanodeOfSignerAttrValue(ctx, signer, attr, value)
   749		if err != nil {
   750			httputil.ServeJSONError(rw, err)
   751			return
   752		}
   753	
   754		dr := h.NewDescribeRequest()
   755		dr.StartDescribe(ctx, pn, 2)
   756		metaMap, err := dr.metaMap()
   757		if err != nil {
   758			httputil.ServeJSONError(rw, err)
   759			return
   760		}
   761	
   762		httputil.ReturnJSON(rw, &SignerAttrValueResponse{
   763			Permanode: pn,
   764			Meta:      metaMap,
   765		})
   766	}
   767	
   768	// EdgesTo returns edges that reference req.RefTo.
   769	// It filters out since-deleted permanode edges.
   770	func (h *Handler) EdgesTo(req *EdgesRequest) (*EdgesResponse, error) {
   771		ctx := context.TODO()
   772		h.index.RLock()
   773		defer h.index.RUnlock()
   774	
   775		toRef := req.ToRef
   776		toRefStr := toRef.String()
   777		var edgeItems []*EdgeItem
   778	
   779		edges, err := h.index.EdgesTo(toRef, nil)
   780		if err != nil {
   781			panic(err)
   782		}
   783	
   784		type edgeOrError struct {
   785			edge *EdgeItem // or nil
   786			err  error
   787		}
   788		resc := make(chan edgeOrError)
   789		verify := func(edge *camtypes.Edge) {
   790			db, err := h.NewDescribeRequest().DescribeSync(ctx, edge.From)
   791			if err != nil {
   792				resc <- edgeOrError{err: err}
   793				return
   794			}
   795			found := false
   796			if db.Permanode != nil {
   797				for attr, vv := range db.Permanode.Attr {
   798					if index.IsBlobReferenceAttribute(attr) {
   799						for _, v := range vv {
   800							if v == toRefStr {
   801								found = true
   802							}
   803						}
   804					}
   805				}
   806			}
   807			var ei *EdgeItem
   808			if found {
   809				ei = &EdgeItem{
   810					From:     edge.From,
   811					FromType: "permanode",
   812				}
   813			}
   814			resc <- edgeOrError{edge: ei}
   815		}
   816		verifying := 0
   817		for _, edge := range edges {
   818			if edge.FromType == "permanode" {
   819				verifying++
   820				go verify(edge)
   821				continue
   822			}
   823			ei := &EdgeItem{
   824				From:     edge.From,
   825				FromType: edge.FromType,
   826			}
   827			edgeItems = append(edgeItems, ei)
   828		}
   829		for i := 0; i < verifying; i++ {
   830			res := <-resc
   831			if res.err != nil {
   832				return nil, res.err
   833			}
   834			if res.edge != nil {
   835				edgeItems = append(edgeItems, res.edge)
   836			}
   837		}
   838	
   839		return &EdgesResponse{
   840			ToRef:   toRef,
   841			EdgesTo: edgeItems,
   842		}, nil
   843	}
   844	
   845	// Unlike the index interface's EdgesTo method, the "edgesto" Handler
   846	// here additionally filters out since-deleted permanode edges.
   847	func (h *Handler) serveEdgesTo(rw http.ResponseWriter, req *http.Request) {
   848		defer httputil.RecoverJSON(rw, req)
   849		var er EdgesRequest
   850		er.fromHTTP(req)
   851		res, err := h.EdgesTo(&er)
   852		if err != nil {
   853			httputil.ServeJSONError(rw, err)
   854			return
   855		}
   856		httputil.ReturnJSON(rw, res)
   857	}
   858	
   859	func (h *Handler) serveQuery(rw http.ResponseWriter, req *http.Request) {
   860		defer httputil.RecoverJSON(rw, req)
   861	
   862		var sq SearchQuery
   863		if err := sq.FromHTTP(req); err != nil {
   864			httputil.ServeJSONError(rw, err)
   865			return
   866		}
   867	
   868		sr, err := h.Query(req.Context(), &sq)
   869		if err != nil {
   870			httputil.ServeJSONError(rw, err)
   871			return
   872		}
   873	
   874		httputil.ReturnJSON(rw, sr)
   875	}
   876	
   877	// GetSignerPaths returns paths with a target of req.Target.
   878	func (h *Handler) GetSignerPaths(req *SignerPathsRequest) (*SignerPathsResponse, error) {
   879		ctx := context.TODO()
   880		if !req.Signer.Valid() {
   881			return nil, errors.New("error getting signer paths: nil signer")
   882		}
   883		if !req.Target.Valid() {
   884			return nil, errors.New("error getting signer paths: nil target")
   885		}
   886		h.index.RLock()
   887		defer h.index.RUnlock()
   888	
   889		paths, err := h.index.PathsOfSignerTarget(ctx, req.Signer, req.Target)
   890		if err != nil {
   891			return nil, fmt.Errorf("Error getting paths of %s: %v", req.Target.String(), err)
   892		}
   893		var jpaths []*SignerPathsItem
   894		for _, path := range paths {
   895			jpaths = append(jpaths, &SignerPathsItem{
   896				ClaimRef: path.Claim,
   897				BaseRef:  path.Base,
   898				Suffix:   path.Suffix,
   899			})
   900		}
   901	
   902		dr := h.NewDescribeRequest()
   903		for _, path := range paths {
   904			dr.StartDescribe(ctx, path.Base, 2)
   905		}
   906		metaMap, err := dr.metaMap()
   907		if err != nil {
   908			return nil, err
   909		}
   910	
   911		res := &SignerPathsResponse{
   912			Paths: jpaths,
   913			Meta:  metaMap,
   914		}
   915		return res, nil
   916	}
   917	
   918	func (h *Handler) serveSignerPaths(rw http.ResponseWriter, req *http.Request) {
   919		defer httputil.RecoverJSON(rw, req)
   920		var sr SignerPathsRequest
   921		sr.fromHTTP(req)
   922	
   923		res, err := h.GetSignerPaths(&sr)
   924		if err != nil {
   925			httputil.ServeJSONError(rw, err)
   926			return
   927		}
   928		httputil.ReturnJSON(rw, res)
   929	}
   930	
   931	// EvalSearchInput checks if its input is JSON. If so it returns a Constraint constructed from that JSON. Otherwise
   932	// it assumes the input to be a search expression. It parses the expression and returns the parsed Constraint.
   933	func evalSearchInput(in string) (*Constraint, error) {
   934		if len(in) == 0 {
   935			return nil, fmt.Errorf("empty expression")
   936		}
   937		if strings.HasPrefix(in, "{") && strings.HasSuffix(in, "}") {
   938			cs := new(Constraint)
   939			if err := json.NewDecoder(strings.NewReader(in)).Decode(&cs); err != nil {
   940				return nil, err
   941			}
   942			return cs, nil
   943		} else {
   944			sq, err := parseExpression(context.TODO(), in)
   945			if err != nil {
   946				return nil, err
   947			}
   948			return sq.Constraint.Logical.B, nil
   949		}
   950	}
   951	
   952	// getNamed displays the search expression or constraint json for the requested alias.
   953	func (sh *Handler) getNamed(ctx context.Context, name string) (string, error) {
   954		if sh.fetcher == nil {
   955			return "", fmt.Errorf("GetNamed functionality not available")
   956		}
   957		sr, err := sh.Query(ctx, NamedSearch(name))
   958		if err != nil {
   959			return "", err
   960		}
   961	
   962		if len(sr.Blobs) < 1 {
   963			return "", fmt.Errorf("No named search found for: %s", name)
   964		}
   965		permaRef := sr.Blobs[0].Blob
   966		substRefS := sr.Describe.Meta.Get(permaRef).Permanode.Attr.Get("camliContent")
   967		br, ok := blob.Parse(substRefS)
   968		if !ok {
   969			return "", fmt.Errorf("Invalid blob ref: %s", substRefS)
   970		}
   971	
   972		reader, _, err := sh.fetcher.Fetch(ctx, br)
   973		if err != nil {
   974			return "", err
   975		}
   976		result, err := ioutil.ReadAll(reader)
   977		if err != nil {
   978			return "", err
   979		}
   980		return string(result), nil
   981	}
   982	
   983	// NamedSearch returns a *SearchQuery to find the permanode of the search alias "name".
   984	func NamedSearch(name string) *SearchQuery {
   985		return &SearchQuery{
   986			Constraint: &Constraint{
   987				Permanode: &PermanodeConstraint{
   988					Attr:  "camliNamedSearch",
   989					Value: name,
   990				},
   991			},
   992			Describe: &DescribeRequest{},
   993		}
   994	}
   995	
   996	const camliTypePrefix = "application/json; camliType="
Website layout inspired by memcached.
Content by the authors.