1	/*
     2	Copyright 2011 The Perkeep Authors
     3	
     4	Licensed under the Apache License, Version 2.0 (the "License");
     5	you may not use this file except in compliance with the License.
     6	You may obtain a copy of the License at
     7	
     8	     http://www.apache.org/licenses/LICENSE-2.0
     9	
    10	Unless required by applicable law or agreed to in writing, software
    11	distributed under the License is distributed on an "AS IS" BASIS,
    12	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13	See the License for the specific language governing permissions and
    14	limitations under the License.
    15	*/
    16	
    17	// Package serverinit is responsible for mapping from a Perkeep
    18	// configuration file and instantiating HTTP Handlers for all the
    19	// necessary endpoints.
    20	package serverinit // import "perkeep.org/pkg/serverinit"
    21	
    22	import (
    23		"bytes"
    24		"context"
    25		"encoding/json"
    26		"errors"
    27		"expvar"
    28		"fmt"
    29		"io"
    30		"log"
    31		"net"
    32		"net/http"
    33		"net/http/pprof"
    34		"os"
    35		"reflect"
    36		"regexp"
    37		"runtime"
    38		"runtime/debug"
    39		rpprof "runtime/pprof"
    40		"strconv"
    41		"strings"
    42	
    43		"perkeep.org/internal/httputil"
    44		"perkeep.org/internal/osutil"
    45		"perkeep.org/pkg/auth"
    46		"perkeep.org/pkg/blobserver"
    47		"perkeep.org/pkg/blobserver/handlers"
    48		"perkeep.org/pkg/index"
    49		"perkeep.org/pkg/jsonsign/signhandler"
    50		"perkeep.org/pkg/server"
    51		"perkeep.org/pkg/server/app"
    52		"perkeep.org/pkg/types/serverconfig"
    53	
    54		"cloud.google.com/go/compute/metadata"
    55		"go4.org/jsonconfig"
    56	)
    57	
    58	const camliPrefix = "/camli/"
    59	
    60	var ErrCamliPath = errors.New("invalid Perkeep request path")
    61	
    62	type handlerConfig struct {
    63		prefix   string         // "/foo/"
    64		htype    string         // "localdisk", etc
    65		conf     jsonconfig.Obj // never nil
    66		internal bool           // if true, not accessible over HTTP
    67	
    68		settingUp, setupDone bool
    69	}
    70	
    71	// handlerLoader implements blobserver.Loader.
    72	type handlerLoader struct {
    73		installer   HandlerInstaller
    74		baseURL     string
    75		config      map[string]*handlerConfig // prefix -> config
    76		handler     map[string]interface{}    // prefix -> http.Handler / func / blobserver.Storage
    77		curPrefix   string
    78		closers     []io.Closer
    79		prefixStack []string
    80	}
    81	
    82	var _ blobserver.Loader = (*handlerLoader)(nil)
    83	
    84	// A HandlerInstaller is anything that can register an HTTP Handler at
    85	// a prefix path.  Both *http.ServeMux and perkeep.org/pkg/webserver.Server
    86	// implement HandlerInstaller.
    87	type HandlerInstaller interface {
    88		Handle(path string, h http.Handler)
    89	}
    90	
    91	type storageAndConfig struct {
    92		blobserver.Storage
    93		config *blobserver.Config
    94	}
    95	
    96	// parseCamliPath looks for "/camli/" in the path and returns
    97	// what follows it (the action).
    98	func parseCamliPath(path string) (action string, err error) {
    99		camIdx := strings.Index(path, camliPrefix)
   100		if camIdx == -1 {
   101			return "", ErrCamliPath
   102		}
   103		action = path[camIdx+len(camliPrefix):]
   104		return
   105	}
   106	
   107	func unsupportedHandler(rw http.ResponseWriter, req *http.Request) {
   108		httputil.BadRequestError(rw, "Unsupported Perkeep path or method.")
   109	}
   110	
   111	func (s *storageAndConfig) Config() *blobserver.Config {
   112		return s.config
   113	}
   114	
   115	// GetStorage returns the unwrapped blobserver.Storage interface value for
   116	// callers to type-assert optional interface implementations on. (e.g. EnumeratorConfig)
   117	func (s *storageAndConfig) GetStorage() blobserver.Storage {
   118		return s.Storage
   119	}
   120	
   121	// action is the part following "/camli/" in the URL. It's either a
   122	// string like "enumerate-blobs", "stat", "upload", or a blobref.
   123	func camliHandlerUsingStorage(req *http.Request, action string, storage blobserver.StorageConfiger) (http.Handler, auth.Operation) {
   124		var handler http.Handler
   125		op := auth.OpAll
   126		switch req.Method {
   127		case "GET", "HEAD":
   128			switch action {
   129			case "enumerate-blobs":
   130				handler = handlers.CreateEnumerateHandler(storage)
   131				op = auth.OpGet
   132			case "stat":
   133				handler = handlers.CreateStatHandler(storage)
   134			case "ws":
   135				handler = nil         // TODO: handlers.CreateSocketHandler(storage)
   136				op = auth.OpDiscovery // rest of operation auth checks done in handler
   137			default:
   138				handler = handlers.CreateGetHandler(storage)
   139				op = auth.OpGet
   140			}
   141		case "POST":
   142			switch action {
   143			case "stat":
   144				handler = handlers.CreateStatHandler(storage)
   145				op = auth.OpStat
   146			case "upload":
   147				handler = handlers.CreateBatchUploadHandler(storage)
   148				op = auth.OpUpload
   149			case "remove":
   150				handler = handlers.CreateRemoveHandler(storage)
   151			}
   152		case "PUT":
   153			handler = handlers.CreatePutUploadHandler(storage)
   154			op = auth.OpUpload
   155		}
   156		if handler == nil {
   157			handler = http.HandlerFunc(unsupportedHandler)
   158		}
   159		return handler, op
   160	}
   161	
   162	// where prefix is like "/" or "/s3/" for e.g. "/camli/" or "/s3/camli/*"
   163	func makeCamliHandler(prefix, baseURL string, storage blobserver.Storage, hf blobserver.FindHandlerByTyper) http.Handler {
   164		if !strings.HasSuffix(prefix, "/") {
   165			panic("expected prefix to end in slash")
   166		}
   167		baseURL = strings.TrimRight(baseURL, "/")
   168	
   169		canLongPoll := true
   170		// TODO(bradfitz): set to false if this is App Engine, or provide some way to disable
   171	
   172		storageConfig := &storageAndConfig{
   173			storage,
   174			&blobserver.Config{
   175				Writable:      true,
   176				Readable:      true,
   177				Deletable:     false,
   178				URLBase:       baseURL + prefix[:len(prefix)-1],
   179				CanLongPoll:   canLongPoll,
   180				HandlerFinder: hf,
   181			},
   182		}
   183		return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
   184			action, err := parseCamliPath(req.URL.Path[len(prefix)-1:])
   185			if err != nil {
   186				log.Printf("Invalid request for method %q, path %q",
   187					req.Method, req.URL.Path)
   188				unsupportedHandler(rw, req)
   189				return
   190			}
   191			handler := auth.RequireAuth(camliHandlerUsingStorage(req, action, storageConfig))
   192			handler.ServeHTTP(rw, req)
   193		})
   194	}
   195	
   196	func (hl *handlerLoader) FindHandlerByType(htype string) (prefix string, handler interface{}, err error) {
   197		nFound := 0
   198		for pfx, config := range hl.config {
   199			if config.htype == htype {
   200				nFound++
   201				prefix, handler = pfx, hl.handler[pfx]
   202			}
   203		}
   204		if nFound == 0 {
   205			return "", nil, blobserver.ErrHandlerTypeNotFound
   206		}
   207		if htype == "jsonsign" && nFound > 1 {
   208			// TODO: do this for all handler types later? audit
   209			// callers of FindHandlerByType and see if that's
   210			// feasible. For now I'm only paranoid about jsonsign.
   211			return "", nil, fmt.Errorf("%d handlers found of type %q; ambiguous", nFound, htype)
   212		}
   213		return
   214	}
   215	
   216	func (hl *handlerLoader) AllHandlers() (types map[string]string, handlers map[string]interface{}) {
   217		types = make(map[string]string)
   218		handlers = make(map[string]interface{})
   219		for pfx, config := range hl.config {
   220			types[pfx] = config.htype
   221			handlers[pfx] = hl.handler[pfx]
   222		}
   223		return
   224	}
   225	
   226	func (hl *handlerLoader) setupAll() {
   227		for prefix := range hl.config {
   228			hl.setupHandler(prefix)
   229		}
   230	}
   231	
   232	func (hl *handlerLoader) configType(prefix string) string {
   233		if h, ok := hl.config[prefix]; ok {
   234			return h.htype
   235		}
   236		return ""
   237	}
   238	
   239	func (hl *handlerLoader) MyPrefix() string {
   240		return hl.curPrefix
   241	}
   242	
   243	func (hl *handlerLoader) BaseURL() string {
   244		return hl.baseURL
   245	}
   246	
   247	func (hl *handlerLoader) GetStorage(prefix string) (blobserver.Storage, error) {
   248		hl.setupHandler(prefix)
   249		if s, ok := hl.handler[prefix].(blobserver.Storage); ok {
   250			return s, nil
   251		}
   252		return nil, fmt.Errorf("bogus storage handler referenced as %q", prefix)
   253	}
   254	
   255	func (hl *handlerLoader) GetHandler(prefix string) (interface{}, error) {
   256		hl.setupHandler(prefix)
   257		if s, ok := hl.handler[prefix].(blobserver.Storage); ok {
   258			return s, nil
   259		}
   260		if h, ok := hl.handler[prefix].(http.Handler); ok {
   261			return h, nil
   262		}
   263		return nil, fmt.Errorf("bogus http or storage handler referenced as %q", prefix)
   264	}
   265	
   266	func (hl *handlerLoader) GetHandlerType(prefix string) string {
   267		return hl.configType(prefix)
   268	}
   269	
   270	func exitFailure(pattern string, args ...interface{}) {
   271		if !strings.HasSuffix(pattern, "\n") {
   272			pattern = pattern + "\n"
   273		}
   274		panic(fmt.Sprintf(pattern, args...))
   275	}
   276	
   277	func (hl *handlerLoader) setupHandler(prefix string) {
   278		h, ok := hl.config[prefix]
   279		if !ok {
   280			exitFailure("invalid reference to undefined handler %q", prefix)
   281		}
   282		if h.setupDone {
   283			// Already setup by something else reference it and forcing it to be
   284			// setup before the bottom loop got to it.
   285			return
   286		}
   287		hl.prefixStack = append(hl.prefixStack, prefix)
   288		if h.settingUp {
   289			buf := make([]byte, 1024)
   290			buf = buf[:runtime.Stack(buf, false)]
   291			exitFailure("loop in configuration graph; %q tried to load itself indirectly: %q\nStack:\n%s",
   292				prefix, hl.prefixStack, buf)
   293		}
   294		h.settingUp = true
   295		defer func() {
   296			// log.Printf("Configured handler %q", prefix)
   297			h.setupDone = true
   298			hl.prefixStack = hl.prefixStack[:len(hl.prefixStack)-1]
   299			r := recover()
   300			if r == nil {
   301				if hl.handler[prefix] == nil {
   302					panic(fmt.Sprintf("setupHandler for %q didn't install a handler", prefix))
   303				}
   304			} else {
   305				panic(r)
   306			}
   307		}()
   308	
   309		hl.curPrefix = prefix
   310	
   311		if strings.HasPrefix(h.htype, "storage-") {
   312			// Assume a storage interface:
   313			stype := strings.TrimPrefix(h.htype, "storage-")
   314			pstorage, err := blobserver.CreateStorage(stype, hl, h.conf)
   315			if err != nil {
   316				exitFailure("error instantiating storage for prefix %q, type %q: %v",
   317					h.prefix, stype, err)
   318			}
   319			if ix, ok := pstorage.(*index.Index); ok && ix.WantsReindex() {
   320				log.Printf("Reindexing %s ...", h.prefix)
   321				if err := ix.Reindex(); err != nil {
   322					if ix.WantsKeepGoing() {
   323						log.Printf("Error reindexing %s: %v", h.prefix, err)
   324					} else {
   325						exitFailure("Error reindexing %s: %v", h.prefix, err)
   326					}
   327				}
   328			}
   329			hl.handler[h.prefix] = pstorage
   330			if h.internal {
   331				hl.installer.Handle(prefix, unauthorizedHandler{})
   332			} else {
   333				hl.installer.Handle(prefix+"camli/", makeCamliHandler(prefix, hl.baseURL, pstorage, hl))
   334			}
   335			if cl, ok := pstorage.(blobserver.ShutdownStorage); ok {
   336				hl.closers = append(hl.closers, cl)
   337			}
   338			return
   339		}
   340	
   341		var hh http.Handler
   342		if h.htype == "app" {
   343			// h.conf might already contain the server's baseURL, but
   344			// perkeepd.go derives (if needed) a more useful hl.baseURL,
   345			// after h.conf was generated, so we provide it as well to
   346			// FromJSONConfig so NewHandler can benefit from it.
   347			hc, err := app.FromJSONConfig(h.conf, prefix, hl.baseURL)
   348			if err != nil {
   349				exitFailure("error setting up app config for prefix %q: %v", h.prefix, err)
   350			}
   351			ap, err := app.NewHandler(hc)
   352			if err != nil {
   353				exitFailure("error setting up app for prefix %q: %v", h.prefix, err)
   354			}
   355			hh = ap
   356			auth.AddMode(ap.AuthMode())
   357			// TODO(mpl): this check is weak, as the user could very well
   358			// use another binary name for the publisher app. We should
   359			// introduce/use another identifier.
   360			if ap.ProgramName() == "publisher" {
   361				if err := hl.initPublisherRootNode(ap); err != nil {
   362					exitFailure("Error looking/setting up root node for publisher on %v: %v", h.prefix, err)
   363				}
   364			}
   365		} else {
   366			var err error
   367			hh, err = blobserver.CreateHandler(h.htype, hl, h.conf)
   368			if err != nil {
   369				exitFailure("error instantiating handler for prefix %q, type %q: %v",
   370					h.prefix, h.htype, err)
   371			}
   372		}
   373	
   374		hl.handler[prefix] = hh
   375		var wrappedHandler http.Handler
   376		if h.internal {
   377			wrappedHandler = unauthorizedHandler{}
   378		} else {
   379			wrappedHandler = &httputil.PrefixHandler{Prefix: prefix, Handler: hh}
   380			if handlerTypeWantsAuth(h.htype) {
   381				wrappedHandler = auth.Handler{Handler: wrappedHandler}
   382			}
   383		}
   384		hl.installer.Handle(prefix, wrappedHandler)
   385	}
   386	
   387	type unauthorizedHandler struct{}
   388	
   389	func (unauthorizedHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
   390		http.Error(w, "Unauthorized", http.StatusUnauthorized)
   391	}
   392	
   393	func handlerTypeWantsAuth(handlerType string) bool {
   394		// TODO(bradfitz): ask the handler instead? This is a bit of a
   395		// weird spot for this policy maybe?
   396		switch handlerType {
   397		case "ui", "search", "jsonsign", "sync", "status", "help", "importer":
   398			return true
   399		}
   400		return false
   401	}
   402	
   403	// A Config is the wrapper around a Perkeep JSON configuration file.
   404	// Files on disk can be in either high-level or low-level format, but
   405	// the Load function always returns the Config in its low-level format
   406	// (a.k.a. the "handler" format).
   407	//
   408	// TODO(bradfitz): document and/or link to the low-level format; for
   409	// now you can see the high-level config format at https://perkeep.org/pkg/types/serverconfig/#Config
   410	// and the the low-level format by running "camtool dumpconfig".
   411	type Config struct {
   412		jconf jsonconfig.Obj // low-level JSON config
   413	
   414		httpsCert  string // optional
   415		httpsKey   string // optional
   416		https      bool
   417		baseURL    string // optional, without trailing slash
   418		listenAddr string // the optional net.Listen-style TCP listen address
   419	
   420		installedHandlers bool   // whether InstallHandlers (which validates the config too) has been called
   421		uiPath            string // Not valid until after InstallHandlers
   422	
   423		// apps is the list of server apps configured during InstallHandlers,
   424		// and that should be started after perkeepd has started serving.
   425		apps []*app.Handler
   426		// signHandler is found and configured during InstallHandlers, or nil.
   427		// It is stored in the Config, so we can call UploadPublicKey on on it as
   428		// soon as perkeepd is ready for it.
   429		signHandler *signhandler.Handler
   430	}
   431	
   432	// UIPath returns the relative path to the server's user interface
   433	// handler, if the UI is configured. Otherwise it returns the empty
   434	// string. It is not valid until after a call to InstallHandlers
   435	//
   436	// If non-empty, the returned value will both begin and end with a slash.
   437	func (c *Config) UIPath() string {
   438		if !c.installedHandlers {
   439			panic("illegal UIPath call before call to InstallHandlers")
   440		}
   441		return c.uiPath
   442	}
   443	
   444	// BaseURL returns the optional URL prefix listening the root of this server.
   445	// It does not end in a trailing slash.
   446	func (c *Config) BaseURL() string { return c.baseURL }
   447	
   448	// ListenAddr returns the optional configured listen address in ":port" or "ip:port" form.
   449	func (c *Config) ListenAddr() string { return c.listenAddr }
   450	
   451	// HTTPSCert returns the optional path to an HTTPS public key certificate file.
   452	func (c *Config) HTTPSCert() string { return c.httpsCert }
   453	
   454	// HTTPSKey returns the optional path to an HTTPS private key file.
   455	func (c *Config) HTTPSKey() string { return c.httpsKey }
   456	
   457	// HTTPS reports whether this configuration wants to serve HTTPS.
   458	func (c *Config) HTTPS() bool { return c.https }
   459	
   460	// IsTailscaleListener reports whether c is configured to run in
   461	// Tailscale tsnet mode.
   462	func (c *Config) IsTailscaleListener() bool {
   463		return c.listenAddr == "tailscale" || strings.HasPrefix(c.listenAddr, "tailscale:")
   464	}
   465	
   466	// detectConfigChange returns an informative error if conf contains obsolete keys.
   467	func detectConfigChange(conf jsonconfig.Obj) error {
   468		oldHTTPSKey, oldHTTPSCert := conf.OptionalString("HTTPSKeyFile", ""), conf.OptionalString("HTTPSCertFile", "")
   469		if oldHTTPSKey != "" || oldHTTPSCert != "" {
   470			return fmt.Errorf("config keys %q and %q have respectively been renamed to %q and %q, please fix your server config",
   471				"HTTPSKeyFile", "HTTPSCertFile", "httpsKey", "httpsCert")
   472		}
   473		return nil
   474	}
   475	
   476	// LoadFile returns a low-level "handler config" from the provided filename.
   477	// If the config file doesn't contain a top-level JSON key of "handlerConfig"
   478	// with boolean value true, the configuration is assumed to be a high-level
   479	// "user config" file, and transformed into a low-level config.
   480	func LoadFile(filename string) (*Config, error) {
   481		return load(filename, nil)
   482	}
   483	
   484	// jsonFileImpl implements jsonconfig.File using a *bytes.Reader with
   485	// the contents slurped into memory.
   486	type jsonFileImpl struct {
   487		*bytes.Reader
   488		name string
   489	}
   490	
   491	func (jsonFileImpl) Close() error   { return nil }
   492	func (f jsonFileImpl) Name() string { return f.name }
   493	
   494	// Load returns a low-level "handler config" from the provided config.
   495	// If the config doesn't contain a top-level JSON key of "handlerConfig"
   496	// with boolean value true, the configuration is assumed to be a high-level
   497	// "user config" file, and transformed into a low-level config.
   498	func Load(config []byte) (*Config, error) {
   499		return load("", func(filename string) (jsonconfig.File, error) {
   500			if filename != "" {
   501				return nil, errors.New("JSON files with includes not supported with jsonconfig.Load")
   502			}
   503			return jsonFileImpl{bytes.NewReader(config), "config file"}, nil
   504		})
   505	}
   506	
   507	func load(filename string, opener func(filename string) (jsonconfig.File, error)) (*Config, error) {
   508		c := osutil.NewJSONConfigParser()
   509		c.Open = opener
   510		m, err := c.ReadFile(filename)
   511		if err != nil {
   512			return nil, err
   513		}
   514		obj := jsonconfig.Obj(m)
   515		conf := &Config{
   516			jconf: obj,
   517		}
   518	
   519		if lowLevel := obj.OptionalBool("handlerConfig", false); lowLevel {
   520			if err := conf.readFields(); err != nil {
   521				return nil, err
   522			}
   523			return conf, nil
   524		}
   525	
   526		// Check whether the high-level config uses the old names.
   527		if err := detectConfigChange(obj); err != nil {
   528			return nil, err
   529		}
   530	
   531		// Because the original high-level config might have expanded
   532		// through the use of functions, we re-encode the map back to
   533		// JSON here so we can unmarshal it into the hiLevelConf
   534		// struct later.
   535		highExpandedJSON, err := json.Marshal(m)
   536		if err != nil {
   537			return nil, fmt.Errorf("Can't re-marshal high-level JSON config: %v", err)
   538		}
   539	
   540		var hiLevelConf serverconfig.Config
   541		if err := json.Unmarshal(highExpandedJSON, &hiLevelConf); err != nil {
   542			return nil, fmt.Errorf("Could not unmarshal into a serverconfig.Config: %v", err)
   543		}
   544	
   545		// At this point, conf.jconf.UnknownKeys() contains all the names found in
   546		// the given high-level configuration. We check them against
   547		// highLevelConfFields(), which gives us all the possible valid
   548		// configuration names, to catch typos or invalid names.
   549		allFields := highLevelConfFields()
   550		for _, v := range conf.jconf.UnknownKeys() {
   551			if _, ok := allFields[v]; !ok {
   552				return nil, fmt.Errorf("unknown high-level configuration parameter: %q in file %q", v, filename)
   553			}
   554		}
   555		conf, err = genLowLevelConfig(&hiLevelConf)
   556		if err != nil {
   557			return nil, fmt.Errorf(
   558				"failed to transform user config file into internal handler configuration: %v",
   559				err)
   560		}
   561		if v, _ := strconv.ParseBool(os.Getenv("CAMLI_DEBUG_CONFIG")); v {
   562			jsconf, _ := json.MarshalIndent(conf.jconf, "", "  ")
   563			log.Printf("From high-level config, generated low-level config: %s", jsconf)
   564		}
   565		if err := conf.readFields(); err != nil {
   566			return nil, err
   567		}
   568		return conf, nil
   569	}
   570	
   571	// readFields reads the low-level jsonconfig fields using the jsonconfig package
   572	// and copies them into c. This marks them as known fields before a future call to InstallerHandlers
   573	func (c *Config) readFields() error {
   574		c.listenAddr = c.jconf.OptionalString("listen", "")
   575		c.baseURL = strings.TrimSuffix(c.jconf.OptionalString("baseURL", ""), "/")
   576		c.httpsCert = c.jconf.OptionalString("httpsCert", "")
   577		c.httpsKey = c.jconf.OptionalString("httpsKey", "")
   578		c.https = c.jconf.OptionalBool("https", false)
   579	
   580		_, explicitHTTPS := c.jconf["https"]
   581		if c.httpsCert != "" && !explicitHTTPS {
   582			return errors.New("httpsCert specified but https was not")
   583		}
   584		if c.httpsKey != "" && !explicitHTTPS {
   585			return errors.New("httpsKey specified but https was not")
   586		}
   587		return nil
   588	}
   589	
   590	func (c *Config) checkValidAuth() error {
   591		authConfig := c.jconf.OptionalString("auth", "")
   592		mode, err := auth.FromConfig(authConfig)
   593		if err == nil {
   594			auth.SetMode(mode)
   595		}
   596		return err
   597	}
   598	
   599	func (c *Config) SetReindex(v bool) {
   600		prefixes, _ := c.jconf["prefixes"].(map[string]interface{})
   601		for prefix, vei := range prefixes {
   602			if prefix == "_knownkeys" {
   603				continue
   604			}
   605			pmap, ok := vei.(map[string]interface{})
   606			if !ok {
   607				continue
   608			}
   609			pconf := jsonconfig.Obj(pmap)
   610			typ, _ := pconf["handler"].(string)
   611			if typ != "storage-index" {
   612				continue
   613			}
   614			opts, ok := pconf["handlerArgs"].(map[string]interface{})
   615			if ok {
   616				opts["reindex"] = v
   617			}
   618		}
   619	}
   620	
   621	// SetKeepGoing changes each configured prefix to set "keepGoing" to true. This
   622	// indicates that validation, reindexing, or recovery behavior should not cause
   623	// the process to end.
   624	func (c *Config) SetKeepGoing(v bool) {
   625		prefixes, _ := c.jconf["prefixes"].(map[string]interface{})
   626		for prefix, vei := range prefixes {
   627			if prefix == "_knownkeys" {
   628				continue
   629			}
   630			pmap, ok := vei.(map[string]interface{})
   631			if !ok {
   632				continue
   633			}
   634			pconf := jsonconfig.Obj(pmap)
   635			typ, _ := pconf["handler"].(string)
   636			if typ != "storage-index" && typ != "storage-blobpacked" {
   637				continue
   638			}
   639			opts, ok := pconf["handlerArgs"].(map[string]interface{})
   640			if ok {
   641				opts["keepGoing"] = v
   642			}
   643		}
   644	}
   645	
   646	// InstallHandlers creates and registers all the HTTP Handlers needed
   647	// by config into the provided HandlerInstaller and validates that the
   648	// configuration is valid.
   649	//
   650	// baseURL is required and specifies the root of this webserver, without trailing slash.
   651	//
   652	// The returned shutdown value can be used to cleanly shut down the
   653	// handlers.
   654	func (c *Config) InstallHandlers(hi HandlerInstaller, baseURL string) (shutdown io.Closer, err error) {
   655		config := c
   656		defer func() {
   657			if e := recover(); e != nil {
   658				log.Printf("Caught panic installer handlers: %v", e)
   659				debug.PrintStack()
   660				err = fmt.Errorf("Caught panic: %v", e)
   661			}
   662		}()
   663	
   664		if err := config.checkValidAuth(); err != nil {
   665			return nil, fmt.Errorf("error while configuring auth: %v", err)
   666		}
   667		prefixes := config.jconf.RequiredObject("prefixes")
   668		if err := config.jconf.Validate(); err != nil {
   669			return nil, fmt.Errorf("configuration error in root object's keys: %v", err)
   670		}
   671	
   672		if v := os.Getenv("CAMLI_PPROF_START"); v != "" {
   673			cpuf := mustCreate(v + ".cpu")
   674			defer cpuf.Close()
   675			memf := mustCreate(v + ".mem")
   676			defer memf.Close()
   677			rpprof.StartCPUProfile(cpuf)
   678			defer rpprof.StopCPUProfile()
   679			defer rpprof.WriteHeapProfile(memf)
   680		}
   681	
   682		hl := &handlerLoader{
   683			installer: hi,
   684			baseURL:   baseURL,
   685			config:    make(map[string]*handlerConfig),
   686			handler:   make(map[string]interface{}),
   687		}
   688	
   689		for prefix, vei := range prefixes {
   690			if prefix == "_knownkeys" {
   691				continue
   692			}
   693			if !strings.HasPrefix(prefix, "/") {
   694				exitFailure("prefix %q doesn't start with /", prefix)
   695			}
   696			if !strings.HasSuffix(prefix, "/") {
   697				exitFailure("prefix %q doesn't end with /", prefix)
   698			}
   699			pmap, ok := vei.(map[string]interface{})
   700			if !ok {
   701				exitFailure("prefix %q value is a %T, not an object", prefix, vei)
   702			}
   703			pconf := jsonconfig.Obj(pmap)
   704			enabled := pconf.OptionalBool("enabled", true)
   705			if !enabled {
   706				continue
   707			}
   708			handlerType := pconf.RequiredString("handler")
   709			handlerArgs := pconf.OptionalObject("handlerArgs")
   710			internal := pconf.OptionalBool("internal", false)
   711			if err := pconf.Validate(); err != nil {
   712				exitFailure("configuration error in prefix %s: %v", prefix, err)
   713			}
   714			h := &handlerConfig{
   715				prefix:   prefix,
   716				htype:    handlerType,
   717				conf:     handlerArgs,
   718				internal: internal,
   719			}
   720			hl.config[prefix] = h
   721	
   722			if handlerType == "ui" {
   723				config.uiPath = prefix
   724			}
   725		}
   726		hl.setupAll()
   727	
   728		// Now that everything is setup, run any handlers' InitHandler
   729		// methods.
   730		// And register apps that will be started later.
   731		for pfx, handler := range hl.handler {
   732			if starter, ok := handler.(*app.Handler); ok {
   733				config.apps = append(config.apps, starter)
   734			}
   735			if helpHandler, ok := handler.(*server.HelpHandler); ok {
   736				helpHandler.SetServerConfig(config.jconf)
   737			}
   738			if signHandler, ok := handler.(*signhandler.Handler); ok {
   739				config.signHandler = signHandler
   740			}
   741			if in, ok := handler.(blobserver.HandlerIniter); ok {
   742				if err := in.InitHandler(hl); err != nil {
   743					return nil, fmt.Errorf("Error calling InitHandler on %s: %v", pfx, err)
   744				}
   745			}
   746		}
   747	
   748		if v, _ := strconv.ParseBool(os.Getenv("CAMLI_HTTP_EXPVAR")); v {
   749			hi.Handle("/debug/vars", expvarHandler{})
   750		}
   751		if v, _ := strconv.ParseBool(os.Getenv("CAMLI_HTTP_PPROF")); v {
   752			hi.Handle("/debug/pprof/", profileHandler{})
   753		}
   754		hi.Handle("/debug/goroutines", auth.RequireAuth(http.HandlerFunc(dumpGoroutines), auth.OpRead))
   755		hi.Handle("/debug/config", auth.RequireAuth(configHandler{config}, auth.OpAll))
   756		hi.Handle("/debug/logs/", auth.RequireAuth(http.HandlerFunc(logsHandler), auth.OpAll))
   757		config.installedHandlers = true
   758		return multiCloser(hl.closers), nil
   759	}
   760	
   761	func dumpGoroutines(w http.ResponseWriter, r *http.Request) {
   762		buf := make([]byte, 2<<20)
   763		buf = buf[:runtime.Stack(buf, true)]
   764		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
   765		w.Write(buf)
   766	}
   767	
   768	// StartApps starts all the server applications that were configured
   769	// during InstallHandlers. It should only be called after perkeepd
   770	// has started serving, since these apps might request some configuration
   771	// from Perkeep to finish initializing.
   772	func (c *Config) StartApps() error {
   773		for _, ap := range c.apps {
   774			if err := ap.Start(); err != nil {
   775				return fmt.Errorf("error starting app %v: %v", ap.ProgramName(), err)
   776			}
   777		}
   778		return nil
   779	}
   780	
   781	// UploadPublicKey uploads the public key blob with the sign handler that was
   782	// configured during InstallHandlers.
   783	func (c *Config) UploadPublicKey(ctx context.Context) error {
   784		if c.signHandler == nil {
   785			return nil
   786		}
   787		return c.signHandler.UploadPublicKey(ctx)
   788	}
   789	
   790	// AppURL returns a map of app name to app base URL for all the configured
   791	// server apps.
   792	func (c *Config) AppURL() map[string]string {
   793		appURL := make(map[string]string, len(c.apps))
   794		for _, ap := range c.apps {
   795			appURL[ap.ProgramName()] = ap.BackendURL()
   796		}
   797		return appURL
   798	}
   799	
   800	func mustCreate(path string) *os.File {
   801		f, err := os.Create(path)
   802		if err != nil {
   803			log.Fatalf("Failed to create %s: %v", path, err)
   804		}
   805		return f
   806	}
   807	
   808	type multiCloser []io.Closer
   809	
   810	func (s multiCloser) Close() (err error) {
   811		for _, cl := range s {
   812			if err1 := cl.Close(); err == nil && err1 != nil {
   813				err = err1
   814			}
   815		}
   816		return
   817	}
   818	
   819	// expvarHandler publishes expvar stats.
   820	type expvarHandler struct{}
   821	
   822	func (expvarHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
   823		w.Header().Set("Content-Type", "application/json; charset=utf-8")
   824		fmt.Fprintf(w, "{\n")
   825		first := true
   826		expvar.Do(func(kv expvar.KeyValue) {
   827			if !first {
   828				fmt.Fprintf(w, ",\n")
   829			}
   830			first = false
   831			fmt.Fprintf(w, "%q: %s", kv.Key, kv.Value)
   832		})
   833		fmt.Fprintf(w, "\n}\n")
   834	}
   835	
   836	type configHandler struct {
   837		c *Config
   838	}
   839	
   840	var (
   841		knownKeys     = regexp.MustCompile(`(?ms)^\s+"_knownkeys": {.+?},?\n`)
   842		sensitiveLine = regexp.MustCompile(`(?m)^\s+\"(auth|aws_secret_access_key|password|client_secret|application_key|passphrase)\": "[^\"]+".*\n`)
   843		trailingComma = regexp.MustCompile(`,(\n\s*\})`)
   844	)
   845	
   846	func (h configHandler) ServeHTTP(w http.ResponseWriter, _ *http.Request) {
   847		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
   848		b, _ := json.MarshalIndent(h.c.jconf, "", "    ")
   849		b = knownKeys.ReplaceAll(b, nil)
   850		b = trailingComma.ReplaceAll(b, []byte("$1"))
   851		b = sensitiveLine.ReplaceAllFunc(b, func(ln []byte) []byte {
   852			i := bytes.IndexByte(ln, ':')
   853			r := string(ln[:i+1]) + ` "REDACTED"`
   854			if bytes.HasSuffix(bytes.TrimSpace(ln), []byte{','}) {
   855				r += ","
   856			}
   857			return []byte(r + "\n")
   858		})
   859		w.Write(b)
   860	}
   861	
   862	// profileHandler publishes server profile information.
   863	type profileHandler struct{}
   864	
   865	func (profileHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
   866		switch req.URL.Path {
   867		case "/debug/pprof/cmdline":
   868			pprof.Cmdline(rw, req)
   869		case "/debug/pprof/profile":
   870			pprof.Profile(rw, req)
   871		case "/debug/pprof/symbol":
   872			pprof.Symbol(rw, req)
   873		default:
   874			pprof.Index(rw, req)
   875		}
   876	}
   877	
   878	func logsHandler(w http.ResponseWriter, r *http.Request) {
   879		suffix := strings.TrimPrefix(r.URL.Path, "/debug/logs/")
   880		switch suffix {
   881		case "perkeepd":
   882			projID, err := metadata.ProjectID()
   883			if err != nil {
   884				httputil.ServeError(w, r, fmt.Errorf("Error getting project ID: %v", err))
   885				return
   886			}
   887			http.Redirect(w, r,
   888				"https://console.developers.google.com/logs?project="+projID+"&service=custom.googleapis.com&logName=perkeepd-stderr",
   889				http.StatusFound)
   890		case "system":
   891			c := &http.Client{
   892				Transport: &http.Transport{
   893					Dial: func(network, addr string) (net.Conn, error) {
   894						return net.Dial("unix", "/run/camjournald.sock")
   895					},
   896				},
   897			}
   898			res, err := c.Get("http://journal/entries")
   899			if err != nil {
   900				http.Error(w, err.Error(), 500)
   901				return
   902			}
   903			w.Header().Set("Content-Type", "text/plain; charset=utf-8")
   904			io.Copy(w, res.Body)
   905		default:
   906			http.Error(w, "no such logs", 404)
   907		}
   908	}
   909	
   910	// highLevelConfFields returns all the possible fields of a serverconfig.Config,
   911	// in their JSON form. This allows checking that the parameters in the high-level
   912	// server configuration file are at least valid names, which is useful to catch
   913	// typos.
   914	func highLevelConfFields() map[string]bool {
   915		knownFields := make(map[string]bool)
   916		var c serverconfig.Config
   917		s := reflect.ValueOf(&c).Elem()
   918		for i := 0; i < s.NumField(); i++ {
   919			f := s.Type().Field(i)
   920			jsonTag, ok := f.Tag.Lookup("json")
   921			if !ok {
   922				panic(fmt.Sprintf("%q field in serverconfig.Config does not have a json tag", f.Name))
   923			}
   924			jsonFields := strings.Split(strings.TrimSuffix(strings.TrimPrefix(jsonTag, `"`), `"`), ",")
   925			jsonName := jsonFields[0]
   926			if jsonName == "" {
   927				panic(fmt.Sprintf("no json field name for %q field in serverconfig.Config", f.Name))
   928			}
   929			knownFields[jsonName] = true
   930		}
   931		return knownFields
   932	}
   933	
   934	// KeyRingAndId returns the GPG identity keyring path and the user's
   935	// GPG keyID (TODO: length/case), if configured. TODO: which error
   936	// value if not configured?
   937	func (c *Config) KeyRingAndId() (keyRing, keyId string, err error) {
   938		prefixes := c.jconf.RequiredObject("prefixes")
   939		if len(prefixes) == 0 {
   940			return "", "", fmt.Errorf("no prefixes object in config")
   941		}
   942		sighelper := prefixes.OptionalObject("/sighelper/")
   943		if len(sighelper) == 0 {
   944			return "", "", fmt.Errorf("no sighelper object in prefixes")
   945		}
   946		handlerArgs := sighelper.OptionalObject("handlerArgs")
   947		if len(handlerArgs) == 0 {
   948			return "", "", fmt.Errorf("no handlerArgs object in sighelper")
   949		}
   950		keyId = handlerArgs.OptionalString("keyId", "")
   951		if keyId == "" {
   952			return "", "", fmt.Errorf("no keyId in sighelper")
   953		}
   954		keyRing = handlerArgs.OptionalString("secretRing", "")
   955		if keyRing == "" {
   956			return "", "", fmt.Errorf("no secretRing in sighelper")
   957		}
   958		return keyRing, keyId, nil
   959	}
   960	
   961	// LowLevelJSONConfig returns the config's underlying low-level JSON form
   962	// for debugging.
   963	//
   964	// Deprecated: this is provided for debugging only and will be going away
   965	// as the move to TOML-based configuration progresses. Do not depend on this.
   966	func (c *Config) LowLevelJSONConfig() map[string]interface{} {
   967		// Make a shallow clone of c.jconf so we can mutate the
   968		// handlerConfig key to make it explicitly true, without
   969		// modifying anybody's else view of it.
   970		ret := map[string]interface{}{}
   971		for k, v := range c.jconf {
   972			ret[k] = v
   973		}
   974		ret["handlerConfig"] = true
   975		return ret
   976	}