1	/*
     2	Copyright 2012 The Perkeep Authors
     3	
     4	Licensed under the Apache License, Version 2.0 (the "License");
     5	you may not use this file except in compliance with the License.
     6	You may obtain a copy of the License at
     7	
     8	     http://www.apache.org/licenses/LICENSE-2.0
     9	
    10	Unless required by applicable law or agreed to in writing, software
    11	distributed under the License is distributed on an "AS IS" BASIS,
    12	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13	See the License for the specific language governing permissions and
    14	limitations under the License.
    15	*/
    16	
    17	package serverinit
    18	
    19	import (
    20		"encoding/json"
    21		"errors"
    22		"fmt"
    23		"log"
    24		"net/url"
    25		"os"
    26		"path"
    27		"path/filepath"
    28		"strconv"
    29		"strings"
    30	
    31		"go4.org/jsonconfig"
    32		"perkeep.org/internal/osutil"
    33		"perkeep.org/pkg/jsonsign"
    34		"perkeep.org/pkg/sorted"
    35		"perkeep.org/pkg/types/serverconfig"
    36	
    37		"go4.org/wkfs"
    38	)
    39	
    40	var (
    41		tempDir = os.TempDir
    42		noMkdir bool // for tests to not call os.Mkdir
    43	)
    44	
    45	type tlsOpts struct {
    46		autoCert  bool // use Perkeep's Let's Encrypt cache. but httpsCert takes precedence, if set.
    47		httpsCert string
    48		httpsKey  string
    49	}
    50	
    51	// genLowLevelConfig returns a low-level config from a high-level config.
    52	func genLowLevelConfig(conf *serverconfig.Config) (lowLevelConf *Config, err error) {
    53		b := &lowBuilder{
    54			high: conf,
    55			low: jsonconfig.Obj{
    56				"prefixes": make(map[string]interface{}),
    57			},
    58		}
    59		return b.build()
    60	}
    61	
    62	// A lowBuilder builds a low-level config from a high-level config.
    63	type lowBuilder struct {
    64		high *serverconfig.Config // high-level config (input)
    65		low  jsonconfig.Obj       // low-level handler config (output)
    66	}
    67	
    68	// args is an alias for map[string]interface{} just to cut down on
    69	// noise below.  But we take care to convert it back to
    70	// map[string]interface{} in the one place where we accept it.
    71	type args map[string]interface{}
    72	
    73	func (b *lowBuilder) addPrefix(at, handler string, a args) {
    74		v := map[string]interface{}{
    75			"handler": handler,
    76		}
    77		if a != nil {
    78			v["handlerArgs"] = (map[string]interface{})(a)
    79		}
    80		b.low["prefixes"].(map[string]interface{})[at] = v
    81	}
    82	
    83	func (b *lowBuilder) hasPrefix(p string) bool {
    84		_, ok := b.low["prefixes"].(map[string]interface{})[p]
    85		return ok
    86	}
    87	
    88	func (b *lowBuilder) runIndex() bool          { return b.high.RunIndex.Get() }
    89	func (b *lowBuilder) copyIndexToMemory() bool { return b.high.CopyIndexToMemory.Get() }
    90	
    91	type dbname string
    92	
    93	// possible arguments to dbName
    94	const (
    95		dbIndex           dbname = "index"
    96		dbBlobpackedIndex dbname = "blobpacked-index"
    97		dbDiskpackedIndex dbname = "diskpacked-index"
    98		dbUIThumbcache    dbname = "ui-thumbcache"
    99		dbSyncQueue       dbname = "queue-sync-to-" // only a prefix. the last part is the sync destination, e.g. "index".
   100	)
   101	
   102	// dbUnique returns the uniqueness string that is used in databases names to
   103	// differentiate them from databases used by other Perkeep instances on the same
   104	// DBMS.
   105	func (b *lowBuilder) dbUnique() string {
   106		if b.high.DBUnique != "" {
   107			return b.high.DBUnique
   108		}
   109		if b.high.Identity != "" {
   110			return strings.ToLower(b.high.Identity)
   111		}
   112		return osutil.Username() // may be empty, if $USER unset
   113	}
   114	
   115	// dbName returns which database to use for the provided user ("of"), which can
   116	// only be one of the const defined above. Returned values all follow the same name
   117	// scheme for consistency:
   118	// -prefixed with "pk_", so as to distinguish them from databases for other programs
   119	// -followed by a username-based uniqueness string
   120	// -last part says which component/part of perkeep it is about
   121	func (b *lowBuilder) dbName(of dbname) string {
   122		unique := b.dbUnique()
   123		if unique == "" {
   124			log.Printf("Could not define uniqueness for database of %q. Do not use the same index DBMS with other Perkeep instances.", of)
   125		}
   126		if unique == useDBNamesConfig {
   127			// this is the hint that we should revert to the old style DBNames, so this
   128			// instance can reuse its existing databases
   129			return b.oldDBNames(of)
   130		}
   131		prefix := "pk_"
   132		if unique != "" {
   133			prefix += unique + "_"
   134		}
   135		switch of {
   136		case dbIndex:
   137			if b.high.DBName != "" {
   138				return b.high.DBName
   139			}
   140			return prefix + "index"
   141		case dbBlobpackedIndex:
   142			return prefix + "blobpacked"
   143		case dbDiskpackedIndex:
   144			return prefix + "diskpacked"
   145		case dbUIThumbcache:
   146			return prefix + "uithumbmeta"
   147		}
   148		asString := string(of)
   149		if strings.HasPrefix(asString, string(dbSyncQueue)) {
   150			return prefix + "syncto_" + strings.TrimPrefix(asString, string(dbSyncQueue))
   151		}
   152		return ""
   153	}
   154	
   155	// As of rev 7eda9fd5027fda88166d6c03b6490cffbf2de5fb, we changed how the
   156	// databases names were defined. But we wanted the existing GCE instances to keep
   157	// on working with the old names, so that nothing would break for existing users,
   158	// without any intervention needed. Through the help of the perkeep-config-version
   159	// variable, set by the GCE launcher, we can know whether an instance is such an
   160	// "old" one, and in that case we keep on using the old database names. oldDBNames
   161	// returns these names.
   162	func (b *lowBuilder) oldDBNames(of dbname) string {
   163		switch of {
   164		case dbIndex:
   165			return "camlistore_index"
   166		case dbBlobpackedIndex:
   167			return "blobpacked_index"
   168		case "queue-sync-to-index":
   169			return "sync_index_queue"
   170		case dbUIThumbcache:
   171			return "ui_thumbmeta_cache"
   172		}
   173		return ""
   174	}
   175	
   176	var errNoOwner = errors.New("no owner")
   177	
   178	// Error is errNoOwner if no identity configured
   179	func (b *lowBuilder) searchOwner() (owner *serverconfig.Owner, err error) {
   180		if b.high.Identity == "" {
   181			return nil, errNoOwner
   182		}
   183		if b.high.IdentitySecretRing == "" {
   184			return nil, errNoOwner
   185		}
   186		return &serverconfig.Owner{
   187			Identity:    b.high.Identity,
   188			SecringFile: b.high.IdentitySecretRing,
   189		}, nil
   190	}
   191	
   192	// longIdentity returns the long form (16 chars) of the GPG key ID, in case the
   193	// user provided the short form (8 chars) or the fingerprint (40 chars) in the config.
   194	func (b *lowBuilder) longIdentity() (string, error) {
   195		if b.high.Identity == "" {
   196			return "", errNoOwner
   197		}
   198		if strings.ToUpper(b.high.Identity) != b.high.Identity {
   199			return "", fmt.Errorf("identity %q is not all upper-case", b.high.Identity)
   200		}
   201		if len(b.high.Identity) == 16 {
   202			return b.high.Identity, nil
   203		}
   204		if len(b.high.Identity) == 40 {
   205			return b.high.Identity[24:], nil
   206		}
   207		if b.high.IdentitySecretRing == "" {
   208			return "", errNoOwner
   209		}
   210		keyID, err := jsonsign.KeyIdFromRing(b.high.IdentitySecretRing)
   211		if err != nil {
   212			return "", fmt.Errorf("could not find any keyID in file %q: %w", b.high.IdentitySecretRing, err)
   213		}
   214		if !strings.HasSuffix(keyID, b.high.Identity) {
   215			return "", fmt.Errorf("%q identity not found in secret ring %v", b.high.Identity, b.high.IdentitySecretRing)
   216		}
   217		return keyID, nil
   218	}
   219	
   220	func addAppConfig(config map[string]interface{}, appConfig *serverconfig.App, low jsonconfig.Obj) {
   221		if appConfig.Listen != "" {
   222			config["listen"] = appConfig.Listen
   223		}
   224		if appConfig.APIHost != "" {
   225			config["apiHost"] = appConfig.APIHost
   226		}
   227		if appConfig.BackendURL != "" {
   228			config["backendURL"] = appConfig.BackendURL
   229		}
   230		if low["listen"] != nil && low["listen"].(string) != "" {
   231			config["serverListen"] = low["listen"].(string)
   232		}
   233		if low["baseURL"] != nil && low["baseURL"].(string) != "" {
   234			config["serverBaseURL"] = low["baseURL"].(string)
   235		}
   236	}
   237	
   238	func (b *lowBuilder) addPublishedConfig(tlsO *tlsOpts) error {
   239		published := b.high.Publish
   240		for k, v := range published {
   241			// trick in case all of the fields of v.App were omitted, which would leave v.App nil.
   242			if v.App == nil {
   243				v.App = &serverconfig.App{}
   244			}
   245			if v.CamliRoot == "" {
   246				return fmt.Errorf("missing \"camliRoot\" key in configuration for %s", k)
   247			}
   248			if v.GoTemplate == "" {
   249				return fmt.Errorf("missing \"goTemplate\" key in configuration for %s", k)
   250			}
   251			appConfig := map[string]interface{}{
   252				"camliRoot":  v.CamliRoot,
   253				"cacheRoot":  v.CacheRoot,
   254				"goTemplate": v.GoTemplate,
   255			}
   256			if v.SourceRoot != "" {
   257				appConfig["sourceRoot"] = v.SourceRoot
   258			}
   259			if v.HTTPSCert != "" && v.HTTPSKey != "" {
   260				// user can specify these directly in the publish section
   261				appConfig["httpsCert"] = v.HTTPSCert
   262				appConfig["httpsKey"] = v.HTTPSKey
   263			} else {
   264				// default to Perkeep parameters, if any
   265				if tlsO != nil {
   266					if tlsO.autoCert {
   267						appConfig["certManager"] = tlsO.autoCert
   268					}
   269					if tlsO.httpsCert != "" {
   270						appConfig["httpsCert"] = tlsO.httpsCert
   271					}
   272					if tlsO.httpsKey != "" {
   273						appConfig["httpsKey"] = tlsO.httpsKey
   274					}
   275				}
   276			}
   277			program := "publisher"
   278			if v.Program != "" {
   279				program = v.Program
   280			}
   281			a := args{
   282				"prefix":    k,
   283				"program":   program,
   284				"appConfig": appConfig,
   285			}
   286			addAppConfig(a, v.App, b.low)
   287			b.addPrefix(k, "app", a)
   288		}
   289		return nil
   290	}
   291	
   292	func (b *lowBuilder) addScanCabConfig(tlsO *tlsOpts) error {
   293		if b.high.ScanCab == nil {
   294			return nil
   295		}
   296		scancab := b.high.ScanCab
   297		if scancab.App == nil {
   298			scancab.App = &serverconfig.App{}
   299		}
   300		if scancab.Prefix == "" {
   301			return errors.New("missing \"prefix\" key in configuration for scanning cabinet")
   302		}
   303	
   304		program := "scanningcabinet"
   305		if scancab.Program != "" {
   306			program = scancab.Program
   307		}
   308	
   309		auth := scancab.Auth
   310		if auth == "" {
   311			auth = b.high.Auth
   312		}
   313		appConfig := map[string]interface{}{
   314			"auth": auth,
   315		}
   316		if scancab.HTTPSCert != "" && scancab.HTTPSKey != "" {
   317			appConfig["httpsCert"] = scancab.HTTPSCert
   318			appConfig["httpsKey"] = scancab.HTTPSKey
   319		} else {
   320			// default to Perkeep parameters, if any
   321			if tlsO != nil {
   322				appConfig["httpsCert"] = tlsO.httpsCert
   323				appConfig["httpsKey"] = tlsO.httpsKey
   324			}
   325		}
   326		a := args{
   327			"prefix":    scancab.Prefix,
   328			"program":   program,
   329			"appConfig": appConfig,
   330		}
   331		addAppConfig(a, scancab.App, b.low)
   332		b.addPrefix(scancab.Prefix, "app", a)
   333		return nil
   334	}
   335	
   336	func (b *lowBuilder) sortedName() string {
   337		switch {
   338		case b.high.MySQL != "":
   339			return "MySQL"
   340		case b.high.PostgreSQL != "":
   341			return "PostgreSQL"
   342		case b.high.Mongo != "":
   343			return "MongoDB"
   344		case b.high.MemoryIndex:
   345			return "in memory LevelDB"
   346		case b.high.SQLite != "":
   347			return "SQLite"
   348		case b.high.KVFile != "":
   349			return "KVFile"
   350		case b.high.LevelDB != "":
   351			return "LevelDB"
   352		}
   353		panic("internal error: sortedName didn't find a sorted implementation")
   354	}
   355	
   356	// kvFileType returns the file based sorted type defined for index storage, if
   357	// any. It defaults to "leveldb" otherwise.
   358	func (b *lowBuilder) kvFileType() string {
   359		switch {
   360		case b.high.SQLite != "":
   361			return "sqlite"
   362		case b.high.KVFile != "":
   363			return "kv"
   364		case b.high.LevelDB != "":
   365			return "leveldb"
   366		default:
   367			return sorted.DefaultKVFileType
   368		}
   369	}
   370	
   371	func (b *lowBuilder) addUIConfig() {
   372		args := map[string]interface{}{
   373			"cache": "/cache/",
   374		}
   375		if b.high.SourceRoot != "" {
   376			args["sourceRoot"] = b.high.SourceRoot
   377		}
   378		var thumbCache map[string]interface{}
   379		if b.high.BlobPath != "" {
   380			thumbCache = map[string]interface{}{
   381				"type": b.kvFileType(),
   382				"file": filepath.Join(b.high.BlobPath, "thumbmeta."+b.kvFileType()),
   383			}
   384		}
   385		if thumbCache == nil {
   386			sorted, err := b.sortedStorage(dbUIThumbcache)
   387			if err == nil {
   388				thumbCache = sorted
   389			}
   390		}
   391		if thumbCache != nil {
   392			args["scaledImage"] = thumbCache
   393		}
   394		b.addPrefix("/ui/", "ui", args)
   395	}
   396	
   397	func (b *lowBuilder) mongoIndexStorage(confStr string, sortedType dbname) (map[string]interface{}, error) {
   398		dbName := b.dbName(sortedType)
   399		if dbName == "" {
   400			return nil, fmt.Errorf("no database name configured for sorted store %q", sortedType)
   401		}
   402		fields := strings.Split(confStr, "@")
   403		if len(fields) == 2 {
   404			host := fields[1]
   405			fields = strings.Split(fields[0], ":")
   406			if len(fields) == 2 {
   407				user, pass := fields[0], fields[1]
   408				return map[string]interface{}{
   409					"type":     "mongo",
   410					"host":     host,
   411					"user":     user,
   412					"password": pass,
   413					"database": dbName,
   414				}, nil
   415			}
   416		}
   417		return nil, errors.New("Malformed mongo config string; want form: \"user:password@host\"")
   418	}
   419	
   420	// parses "user@host:password", which you think would be easy, but we
   421	// documented this format without thinking about port numbers, so this
   422	// uses heuristics to guess what extra colons mean.
   423	func parseUserHostPass(v string) (user, host, password string, ok bool) {
   424		f := strings.SplitN(v, "@", 2)
   425		if len(f) != 2 {
   426			return
   427		}
   428		user = f[0]
   429		f = strings.Split(f[1], ":")
   430		if len(f) < 2 {
   431			return "", "", "", false
   432		}
   433		host = f[0]
   434		f = f[1:]
   435		if len(f) >= 2 {
   436			if _, err := strconv.ParseUint(f[0], 10, 16); err == nil {
   437				host = host + ":" + f[0]
   438				f = f[1:]
   439			}
   440		}
   441		password = strings.Join(f, ":")
   442		ok = true
   443		return
   444	}
   445	
   446	func (b *lowBuilder) dbIndexStorage(rdbms, confStr string, sortedType dbname) (map[string]interface{}, error) {
   447		dbName := b.dbName(sortedType)
   448		if dbName == "" {
   449			return nil, fmt.Errorf("no database name configured for sorted store %q", sortedType)
   450		}
   451		user, host, password, ok := parseUserHostPass(confStr)
   452		if !ok {
   453			return nil, fmt.Errorf("Malformed %s config string. Want: \"user@host:password\"", rdbms)
   454		}
   455		return map[string]interface{}{
   456			"type":     rdbms,
   457			"host":     host,
   458			"user":     user,
   459			"password": password,
   460			"database": dbName,
   461		}, nil
   462	}
   463	
   464	func (b *lowBuilder) sortedStorage(sortedType dbname) (map[string]interface{}, error) {
   465		return b.sortedStorageAt(sortedType, "")
   466	}
   467	
   468	// sortedDBMS returns the configuration for a name database on one of the
   469	// DBMS, if any was found in the configuration. It returns nil otherwise.
   470	func (b *lowBuilder) sortedDBMS(named dbname) (map[string]interface{}, error) {
   471		if b.high.MySQL != "" {
   472			return b.dbIndexStorage("mysql", b.high.MySQL, named)
   473		}
   474		if b.high.PostgreSQL != "" {
   475			return b.dbIndexStorage("postgres", b.high.PostgreSQL, named)
   476		}
   477		if b.high.Mongo != "" {
   478			return b.mongoIndexStorage(b.high.Mongo, named)
   479		}
   480		return nil, nil
   481	}
   482	
   483	// filePrefix gives a file path of where to put the database. It can be omitted by
   484	// some sorted implementations, but is required by others.
   485	// The filePrefix should be to a file, not a directory, and should not end in a ".ext" extension.
   486	// An extension like ".kv" or ".sqlite" will be added.
   487	func (b *lowBuilder) sortedStorageAt(sortedType dbname, filePrefix string) (map[string]interface{}, error) {
   488		dbms, err := b.sortedDBMS(sortedType)
   489		if err != nil {
   490			return nil, err
   491		}
   492		if dbms != nil {
   493			return dbms, nil
   494		}
   495		if b.high.MemoryIndex {
   496			return map[string]interface{}{
   497				"type": "memory",
   498			}, nil
   499		}
   500		if sortedType != "index" && filePrefix == "" {
   501			return nil, fmt.Errorf("internal error: use of sortedStorageAt with a non-index type (%v) and no file location for non-database sorted implementation", sortedType)
   502		}
   503		// dbFile returns path directly if sortedType == "index", else it returns filePrefix+"."+ext.
   504		dbFile := func(path, ext string) string {
   505			if sortedType == "index" {
   506				return path
   507			}
   508			return filePrefix + "." + ext
   509		}
   510		if b.high.SQLite != "" {
   511			return map[string]interface{}{
   512				"type": "sqlite",
   513				"file": dbFile(b.high.SQLite, "sqlite"),
   514			}, nil
   515		}
   516		if b.high.KVFile != "" {
   517			return map[string]interface{}{
   518				"type": "kv",
   519				"file": dbFile(b.high.KVFile, "kv"),
   520			}, nil
   521		}
   522		if b.high.LevelDB != "" {
   523			return map[string]interface{}{
   524				"type": "leveldb",
   525				"file": dbFile(b.high.LevelDB, "leveldb"),
   526			}, nil
   527		}
   528		panic("internal error: sortedStorageAt didn't find a sorted implementation")
   529	}
   530	
   531	func (b *lowBuilder) thatQueueUnlessMemory(thatQueue map[string]interface{}) (queue map[string]interface{}) {
   532		// TODO(mpl): what about if b.high.MemoryIndex ?
   533		if b.high.MemoryStorage {
   534			return map[string]interface{}{
   535				"type": "memory",
   536			}
   537		}
   538		return thatQueue
   539	}
   540	
   541	func (b *lowBuilder) addS3Config(s3 string, vendor string) error {
   542		f := strings.SplitN(s3, ":", 4)
   543		if len(f) < 3 {
   544			m := fmt.Sprintf(`genconfig: expected "%s" field to be of form "access_key_id:secret_access_key:bucket[/optional/dir][:hostname]"`, vendor)
   545			return errors.New(m)
   546		}
   547		accessKey, secret, bucket := f[0], f[1], f[2]
   548		var hostname string
   549		if len(f) == 4 {
   550			hostname = f[3]
   551		}
   552		isReplica := b.hasPrefix("/bs/")
   553		s3Prefix := "/bs/"
   554		if isReplica {
   555			s3Prefix = fmt.Sprintf("/sto-%s/", vendor)
   556		}
   557	
   558		s3Args := func(bucket string) args {
   559			a := args{
   560				"bucket":                bucket,
   561				"aws_access_key":        accessKey,
   562				"aws_secret_access_key": secret,
   563			}
   564			if hostname != "" {
   565				a["hostname"] = hostname
   566			}
   567			return a
   568		}
   569	
   570		if !b.high.PackRelated {
   571			b.addPrefix(s3Prefix, "storage-s3", s3Args(bucket))
   572		} else {
   573			bsLoose := "/bs-loose/"
   574			bsPacked := "/bs-packed/"
   575			if isReplica {
   576				bsLoose = fmt.Sprintf("/sto-%s-bs-loose/", vendor)
   577				bsPacked = fmt.Sprintf("/sto-%s-bs-packed/", vendor)
   578			}
   579	
   580			b.addPrefix(bsLoose, "storage-s3", s3Args(path.Join(bucket, "loose")))
   581			b.addPrefix(bsPacked, "storage-s3", s3Args(path.Join(bucket, "packed")))
   582	
   583			// If index is DBMS, then blobPackedIndex is in DBMS too.
   584			// Otherwise blobPackedIndex is same file-based DB as the index,
   585			// in same dir, but named packindex.dbtype.
   586			blobPackedIndex, err := b.sortedStorageAt(dbBlobpackedIndex, filepath.Join(b.indexFileDir(), "packindex"))
   587			if err != nil {
   588				return err
   589			}
   590			b.addPrefix(s3Prefix, "storage-blobpacked", args{
   591				"smallBlobs": bsLoose,
   592				"largeBlobs": bsPacked,
   593				"metaIndex":  blobPackedIndex,
   594			})
   595		}
   596	
   597		if isReplica {
   598			if b.high.BlobPath == "" && !b.high.MemoryStorage {
   599				panic("unexpected empty blobpath with sync-to-s3")
   600			}
   601			p := fmt.Sprintf("/sync-to-%s/", vendor)
   602			queue := fmt.Sprintf("sync-to-%s-queue.", vendor)
   603			b.addPrefix(p, "sync", args{
   604				"from": "/bs/",
   605				"to":   s3Prefix,
   606				"queue": b.thatQueueUnlessMemory(
   607					map[string]interface{}{
   608						"type": b.kvFileType(),
   609						"file": filepath.Join(b.high.BlobPath, queue+b.kvFileType()),
   610					}),
   611			})
   612			return nil
   613		}
   614	
   615		// TODO(mpl): s3CacheBucket
   616		// See https://perkeep.org/issue/85
   617		b.addPrefix("/cache/", "storage-filesystem", args{
   618			"path": filepath.Join(tempDir(), "camli-cache"),
   619		})
   620	
   621		return nil
   622	}
   623	
   624	func (b *lowBuilder) addB2Config(b2 string) error {
   625		return b.addS3Config(b2, "b2")
   626	}
   627	
   628	func (b *lowBuilder) addGoogleDriveConfig(v string) error {
   629		f := strings.SplitN(v, ":", 4)
   630		if len(f) != 4 {
   631			return errors.New(`genconfig: expected "googledrive" field to be of form "client_id:client_secret:refresh_token:parent_id"`)
   632		}
   633		clientId, secret, refreshToken, parentId := f[0], f[1], f[2], f[3]
   634	
   635		isPrimary := !b.hasPrefix("/bs/")
   636		prefix := ""
   637		if isPrimary {
   638			prefix = "/bs/"
   639			if b.high.PackRelated {
   640				return errors.New("TODO: finish packRelated support for Google Drive")
   641			}
   642		} else {
   643			prefix = "/sto-googledrive/"
   644		}
   645		b.addPrefix(prefix, "storage-googledrive", args{
   646			"parent_id": parentId,
   647			"auth": map[string]interface{}{
   648				"client_id":     clientId,
   649				"client_secret": secret,
   650				"refresh_token": refreshToken,
   651			},
   652		})
   653	
   654		if isPrimary {
   655			b.addPrefix("/cache/", "storage-filesystem", args{
   656				"path": filepath.Join(tempDir(), "camli-cache"),
   657			})
   658		} else {
   659			b.addPrefix("/sync-to-googledrive/", "sync", args{
   660				"from": "/bs/",
   661				"to":   prefix,
   662				"queue": b.thatQueueUnlessMemory(
   663					map[string]interface{}{
   664						"type": b.kvFileType(),
   665						"file": filepath.Join(b.high.BlobPath, "sync-to-googledrive-queue."+b.kvFileType()),
   666					}),
   667			})
   668		}
   669	
   670		return nil
   671	}
   672	
   673	var errGCSUsage = errors.New(`genconfig: expected "googlecloudstorage" field to be of form "client_id:client_secret:refresh_token:bucket[/dir/]" or ":bucketname[/dir/]"`)
   674	
   675	func (b *lowBuilder) addGoogleCloudStorageConfig(v string) error {
   676		var clientID, secret, refreshToken, bucket string
   677		f := strings.SplitN(v, ":", 4)
   678		switch len(f) {
   679		default:
   680			return errGCSUsage
   681		case 4:
   682			clientID, secret, refreshToken, bucket = f[0], f[1], f[2], f[3]
   683		case 2:
   684			if f[0] != "" {
   685				return errGCSUsage
   686			}
   687			bucket = f[1]
   688			clientID = "auto"
   689		}
   690	
   691		isReplica := b.hasPrefix("/bs/")
   692		gsPrefix := "/bs/"
   693		if isReplica {
   694			gsPrefix = "/sto-googlecloudstorage/"
   695		}
   696	
   697		gsArgs := func(bucket string) args {
   698			a := args{
   699				"bucket": bucket,
   700				"auth": map[string]interface{}{
   701					"client_id":     clientID,
   702					"client_secret": secret,
   703					"refresh_token": refreshToken,
   704				},
   705			}
   706			return a
   707		}
   708	
   709		if !b.high.PackRelated {
   710			b.addPrefix(gsPrefix, "storage-googlecloudstorage", gsArgs(bucket))
   711		} else {
   712			bsLoose := "/bs-loose/"
   713			bsPacked := "/bs-packed/"
   714			if isReplica {
   715				bsLoose = "/sto-googlecloudstorage-bs-loose/"
   716				bsPacked = "/sto-googlecloudstorage-bs-packed/"
   717			}
   718	
   719			b.addPrefix(bsLoose, "storage-googlecloudstorage", gsArgs(path.Join(bucket, "loose")))
   720			b.addPrefix(bsPacked, "storage-googlecloudstorage", gsArgs(path.Join(bucket, "packed")))
   721	
   722			// If index is DBMS, then blobPackedIndex is in DBMS too.
   723			// Otherwise blobPackedIndex is same file-based DB as the index,
   724			// in same dir, but named packindex.dbtype.
   725			blobPackedIndex, err := b.sortedStorageAt(dbBlobpackedIndex, filepath.Join(b.indexFileDir(), "packindex"))
   726			if err != nil {
   727				return err
   728			}
   729			b.addPrefix(gsPrefix, "storage-blobpacked", args{
   730				"smallBlobs": bsLoose,
   731				"largeBlobs": bsPacked,
   732				"metaIndex":  blobPackedIndex,
   733			})
   734		}
   735	
   736		if isReplica {
   737			if b.high.BlobPath == "" && !b.high.MemoryStorage {
   738				panic("unexpected empty blobpath with sync-to-googlecloudstorage")
   739			}
   740			b.addPrefix("/sync-to-googlecloudstorage/", "sync", args{
   741				"from": "/bs/",
   742				"to":   gsPrefix,
   743				"queue": b.thatQueueUnlessMemory(
   744					map[string]interface{}{
   745						"type": b.kvFileType(),
   746						"file": filepath.Join(b.high.BlobPath, "sync-to-googlecloud-queue."+b.kvFileType()),
   747					}),
   748			})
   749			return nil
   750		}
   751	
   752		// TODO: cacheBucket like s3CacheBucket?
   753		b.addPrefix("/cache/", "storage-filesystem", args{
   754			"path": filepath.Join(tempDir(), "camli-cache"),
   755		})
   756	
   757		return nil
   758	}
   759	
   760	// indexFileDir returns the directory of the sqlite or kv file, or the
   761	// empty string.
   762	func (b *lowBuilder) indexFileDir() string {
   763		switch {
   764		case b.high.SQLite != "":
   765			return filepath.Dir(b.high.SQLite)
   766		case b.high.KVFile != "":
   767			return filepath.Dir(b.high.KVFile)
   768		case b.high.LevelDB != "":
   769			return filepath.Dir(b.high.LevelDB)
   770		}
   771		return ""
   772	}
   773	
   774	func (b *lowBuilder) syncToIndexArgs() (map[string]interface{}, error) {
   775		a := map[string]interface{}{
   776			"from": "/bs/",
   777			"to":   "/index/",
   778		}
   779	
   780		// TODO(mpl): see if we want to have the same logic with all the other queues. probably.
   781		const sortedType = "queue-sync-to-index"
   782		if dbName := b.dbName(sortedType); dbName != "" {
   783			qj, err := b.sortedDBMS(sortedType)
   784			if err != nil {
   785				return nil, err
   786			}
   787			if qj == nil && b.high.MemoryIndex {
   788				qj = map[string]interface{}{
   789					"type": "memory",
   790				}
   791			}
   792			if qj != nil {
   793				// i.e. the index is configured on a DBMS, so we put the queue there too
   794				a["queue"] = qj
   795				return a, nil
   796			}
   797		}
   798	
   799		// TODO: currently when using s3, the index must be
   800		// sqlite or kvfile, since only through one of those
   801		// can we get a directory.
   802		if !b.high.MemoryStorage && b.high.BlobPath == "" && b.indexFileDir() == "" {
   803			// We don't actually have a working sync handler, but we keep a stub registered
   804			// so it can be referred to from other places.
   805			// See http://perkeep.org/issue/201
   806			a["idle"] = true
   807			return a, nil
   808		}
   809	
   810		dir := b.high.BlobPath
   811		if dir == "" {
   812			dir = b.indexFileDir()
   813		}
   814		a["queue"] = b.thatQueueUnlessMemory(
   815			map[string]interface{}{
   816				"type": b.kvFileType(),
   817				"file": filepath.Join(dir, "sync-to-index-queue."+b.kvFileType()),
   818			})
   819	
   820		return a, nil
   821	}
   822	
   823	func (b *lowBuilder) genLowLevelPrefixes() error {
   824		root := "/bs/"
   825		pubKeyDest := root
   826		if b.runIndex() {
   827			root = "/bs-and-maybe-also-index/"
   828			pubKeyDest = "/bs-and-index/"
   829		}
   830	
   831		rootArgs := map[string]interface{}{
   832			"stealth":      false,
   833			"blobRoot":     root,
   834			"helpRoot":     "/help/",
   835			"statusRoot":   "/status/",
   836			"jsonSignRoot": "/sighelper/",
   837		}
   838		if b.high.OwnerName != "" {
   839			rootArgs["ownerName"] = b.high.OwnerName
   840		}
   841		if b.runIndex() {
   842			rootArgs["searchRoot"] = "/my-search/"
   843		}
   844		if path := b.high.ShareHandlerPath; path != "" {
   845			rootArgs["shareRoot"] = path
   846			b.addPrefix(path, "share", args{
   847				"blobRoot": "/bs/",
   848				"index":    "/index/",
   849			})
   850		}
   851		b.addPrefix("/", "root", rootArgs)
   852		b.addPrefix("/status/", "status", nil)
   853		b.addPrefix("/help/", "help", nil)
   854	
   855		importerArgs := args{}
   856		if b.high.Flickr != "" {
   857			importerArgs["flickr"] = map[string]interface{}{
   858				"clientSecret": b.high.Flickr,
   859			}
   860		}
   861		if b.high.Picasa != "" {
   862			importerArgs["picasa"] = map[string]interface{}{
   863				"clientSecret": b.high.Picasa,
   864			}
   865		}
   866		if b.high.Instapaper != "" {
   867			importerArgs["instapaper"] = map[string]interface{}{
   868				"clientSecret": b.high.Instapaper,
   869			}
   870		}
   871		if b.runIndex() {
   872			b.addPrefix("/importer/", "importer", importerArgs)
   873		}
   874	
   875		b.addPrefix("/sighelper/", "jsonsign", args{
   876			"secretRing":    b.high.IdentitySecretRing,
   877			"keyId":         b.high.Identity,
   878			"publicKeyDest": pubKeyDest,
   879		})
   880	
   881		storageType := "filesystem"
   882		if b.high.PackBlobs {
   883			storageType = "diskpacked"
   884		}
   885		if b.high.BlobPath != "" {
   886			if b.high.PackRelated {
   887				b.addPrefix("/bs-loose/", "storage-filesystem", args{
   888					"path": b.high.BlobPath,
   889				})
   890				b.addPrefix("/bs-packed/", "storage-filesystem", args{
   891					"path": filepath.Join(b.high.BlobPath, "packed"),
   892				})
   893				blobPackedIndex, err := b.sortedStorageAt(dbBlobpackedIndex, filepath.Join(b.high.BlobPath, "packed", "packindex"))
   894				if err != nil {
   895					return err
   896				}
   897				b.addPrefix("/bs/", "storage-blobpacked", args{
   898					"smallBlobs": "/bs-loose/",
   899					"largeBlobs": "/bs-packed/",
   900					"metaIndex":  blobPackedIndex,
   901				})
   902			} else if b.high.PackBlobs {
   903				diskpackedIndex, err := b.sortedStorageAt(dbDiskpackedIndex, filepath.Join(b.high.BlobPath, "diskpacked-index"))
   904				if err != nil {
   905					return err
   906				}
   907				b.addPrefix("/bs/", "storage-"+storageType, args{
   908					"path":      b.high.BlobPath,
   909					"metaIndex": diskpackedIndex,
   910				})
   911			} else {
   912				b.addPrefix("/bs/", "storage-"+storageType, args{
   913					"path": b.high.BlobPath,
   914				})
   915			}
   916			if b.high.PackBlobs {
   917				b.addPrefix("/cache/", "storage-"+storageType, args{
   918					"path": filepath.Join(b.high.BlobPath, "/cache"),
   919					"metaIndex": map[string]interface{}{
   920						"type": b.kvFileType(),
   921						"file": filepath.Join(b.high.BlobPath, "cache", "index."+b.kvFileType()),
   922					},
   923				})
   924			} else {
   925				b.addPrefix("/cache/", "storage-"+storageType, args{
   926					"path": filepath.Join(b.high.BlobPath, "/cache"),
   927				})
   928			}
   929		} else if b.high.MemoryStorage {
   930			b.addPrefix("/bs/", "storage-memory", nil)
   931			b.addPrefix("/cache/", "storage-memory", nil)
   932		}
   933	
   934		if b.runIndex() {
   935			syncArgs, err := b.syncToIndexArgs()
   936			if err != nil {
   937				return err
   938			}
   939			b.addPrefix("/sync/", "sync", syncArgs)
   940	
   941			b.addPrefix("/bs-and-index/", "storage-replica", args{
   942				"backends": []interface{}{"/bs/", "/index/"},
   943			})
   944	
   945			b.addPrefix("/bs-and-maybe-also-index/", "storage-cond", args{
   946				"write": map[string]interface{}{
   947					"if":   "isSchema",
   948					"then": "/bs-and-index/",
   949					"else": "/bs/",
   950				},
   951				"read": "/bs/",
   952			})
   953	
   954			owner, err := b.searchOwner()
   955			if err != nil {
   956				return err
   957			}
   958			searchArgs := args{
   959				"index": "/index/",
   960				"owner": map[string]interface{}{
   961					"identity":    owner.Identity,
   962					"secringFile": owner.SecringFile,
   963				},
   964			}
   965			if b.copyIndexToMemory() {
   966				searchArgs["slurpToMemory"] = true
   967			}
   968			b.addPrefix("/my-search/", "search", searchArgs)
   969		}
   970	
   971		return nil
   972	}
   973	
   974	func (b *lowBuilder) build() (*Config, error) {
   975		conf, low := b.high, b.low
   976		if conf.HTTPS {
   977			if (conf.HTTPSCert != "") != (conf.HTTPSKey != "") {
   978				return nil, errors.New("Must set both httpsCert and httpsKey (or neither to generate a self-signed cert)")
   979			}
   980			if conf.HTTPSCert != "" {
   981				low["httpsCert"] = conf.HTTPSCert
   982				low["httpsKey"] = conf.HTTPSKey
   983			}
   984		}
   985	
   986		if conf.BaseURL != "" {
   987			u, err := url.Parse(conf.BaseURL)
   988			if err != nil {
   989				return nil, fmt.Errorf("Error parsing baseURL %q as a URL: %w", conf.BaseURL, err)
   990			}
   991			if u.Path != "" && u.Path != "/" {
   992				return nil, fmt.Errorf("baseURL can't have a path, only a scheme, host, and optional port")
   993			}
   994			u.Path = ""
   995			low["baseURL"] = u.String()
   996		}
   997		if conf.Listen != "" {
   998			low["listen"] = conf.Listen
   999		}
  1000		if conf.PackBlobs && conf.PackRelated {
  1001			return nil, errors.New("can't use both packBlobs (for 'diskpacked') and packRelated (for 'blobpacked')")
  1002		}
  1003		low["https"] = conf.HTTPS
  1004		low["auth"] = conf.Auth
  1005	
  1006		numIndexers := numSet(conf.LevelDB, conf.Mongo, conf.MySQL, conf.PostgreSQL, conf.SQLite, conf.KVFile, conf.MemoryIndex)
  1007	
  1008		switch {
  1009		case b.runIndex() && numIndexers == 0:
  1010			return nil, fmt.Errorf("Unless runIndex is set to false, you must specify an index option (kvIndexFile, leveldb, mongo, mysql, postgres, sqlite, memoryIndex).")
  1011		case b.runIndex() && numIndexers != 1:
  1012			return nil, fmt.Errorf("With runIndex set true, you can only pick exactly one indexer (mongo, mysql, postgres, sqlite, kvIndexFile, leveldb, memoryIndex).")
  1013		case !b.runIndex() && numIndexers != 0:
  1014			log.Printf("Indexer disabled, but %v will be used for other indexes, queues, caches, etc.", b.sortedName())
  1015		}
  1016	
  1017		longID, err := b.longIdentity()
  1018		if err != nil {
  1019			return nil, err
  1020		}
  1021		b.high.Identity = longID
  1022	
  1023		noLocalDisk := conf.BlobPath == ""
  1024		if noLocalDisk {
  1025			if !conf.MemoryStorage && conf.S3 == "" && conf.B2 == "" && conf.GoogleCloudStorage == "" {
  1026				return nil, errors.New("Unless memoryStorage is set, you must specify at least one storage option for your blobserver (blobPath (for localdisk), s3, b2, googlecloudstorage).")
  1027			}
  1028			if !conf.MemoryStorage && conf.S3 != "" && conf.GoogleCloudStorage != "" {
  1029				return nil, errors.New("Using S3 as a primary storage and Google Cloud Storage as a mirror is not supported for now.")
  1030			}
  1031			if !conf.MemoryStorage && conf.B2 != "" && conf.GoogleCloudStorage != "" {
  1032				return nil, errors.New("Using B2 as a primary storage and Google Cloud Storage as a mirror is not supported for now.")
  1033			}
  1034		}
  1035		if conf.ShareHandler && conf.ShareHandlerPath == "" {
  1036			conf.ShareHandlerPath = "/share/"
  1037		}
  1038		if conf.MemoryStorage {
  1039			noMkdir = true
  1040			if conf.BlobPath != "" {
  1041				return nil, errors.New("memoryStorage and blobPath are mutually exclusive.")
  1042			}
  1043			if conf.PackRelated {
  1044				return nil, errors.New("memoryStorage doesn't support packRelated.")
  1045			}
  1046		}
  1047	
  1048		if err := b.genLowLevelPrefixes(); err != nil {
  1049			return nil, err
  1050		}
  1051	
  1052		var cacheDir string
  1053		if noLocalDisk {
  1054			// Whether perkeepd is run from EC2 or not, we use
  1055			// a temp dir as the cache when primary storage is S3.
  1056			// TODO(mpl): s3CacheBucket
  1057			// See https://perkeep.org/issue/85
  1058			cacheDir = filepath.Join(tempDir(), "camli-cache")
  1059		} else {
  1060			cacheDir = filepath.Join(conf.BlobPath, "cache")
  1061		}
  1062		if !noMkdir {
  1063			if err := os.MkdirAll(cacheDir, 0700); err != nil {
  1064				return nil, fmt.Errorf("Could not create blobs cache dir %s: %w", cacheDir, err)
  1065			}
  1066		}
  1067	
  1068		if len(conf.Publish) > 0 {
  1069			if !b.runIndex() {
  1070				return nil, fmt.Errorf("publishing requires an index")
  1071			}
  1072			var tlsO *tlsOpts
  1073			httpsCert, ok1 := low["httpsCert"].(string)
  1074			httpsKey, ok2 := low["httpsKey"].(string)
  1075			if ok1 && ok2 {
  1076				tlsO = &tlsOpts{
  1077					httpsCert: httpsCert,
  1078					httpsKey:  httpsKey,
  1079				}
  1080			} else if conf.HTTPS {
  1081				tlsO = &tlsOpts{
  1082					autoCert: true,
  1083				}
  1084			}
  1085			if err := b.addPublishedConfig(tlsO); err != nil {
  1086				return nil, fmt.Errorf("Could not generate config for published: %w", err)
  1087			}
  1088		}
  1089	
  1090		if conf.ScanCab != nil {
  1091			if !b.runIndex() {
  1092				return nil, fmt.Errorf("scanning cabinet requires an index")
  1093			}
  1094			var tlsO *tlsOpts
  1095			httpsCert, ok1 := low["httpsCert"].(string)
  1096			httpsKey, ok2 := low["httpsKey"].(string)
  1097			if ok1 && ok2 {
  1098				tlsO = &tlsOpts{
  1099					httpsCert: httpsCert,
  1100					httpsKey:  httpsKey,
  1101				}
  1102			}
  1103			if err := b.addScanCabConfig(tlsO); err != nil {
  1104				return nil, fmt.Errorf("Could not generate config for scanning cabinet: %w", err)
  1105			}
  1106		}
  1107	
  1108		if b.runIndex() {
  1109			b.addUIConfig()
  1110			sto, err := b.sortedStorage("index")
  1111			if err != nil {
  1112				return nil, err
  1113			}
  1114			b.addPrefix("/index/", "storage-index", args{
  1115				"blobSource": "/bs/",
  1116				"storage":    sto,
  1117			})
  1118		}
  1119	
  1120		if conf.S3 != "" {
  1121			if err := b.addS3Config(conf.S3, "s3"); err != nil {
  1122				return nil, err
  1123			}
  1124		}
  1125		if conf.B2 != "" {
  1126			if err := b.addB2Config(conf.B2); err != nil {
  1127				return nil, err
  1128			}
  1129		}
  1130		if conf.GoogleDrive != "" {
  1131			if err := b.addGoogleDriveConfig(conf.GoogleDrive); err != nil {
  1132				return nil, err
  1133			}
  1134		}
  1135		if conf.GoogleCloudStorage != "" {
  1136			if err := b.addGoogleCloudStorageConfig(conf.GoogleCloudStorage); err != nil {
  1137				return nil, err
  1138			}
  1139		}
  1140	
  1141		return &Config{jconf: b.low}, nil
  1142	}
  1143	
  1144	func numSet(vv ...interface{}) (num int) {
  1145		for _, vi := range vv {
  1146			switch v := vi.(type) {
  1147			case string:
  1148				if v != "" {
  1149					num++
  1150				}
  1151			case bool:
  1152				if v {
  1153					num++
  1154				}
  1155			default:
  1156				panic("unknown type")
  1157			}
  1158		}
  1159		return
  1160	}
  1161	
  1162	var defaultBaseConfig = serverconfig.Config{
  1163		Listen: ":3179",
  1164		HTTPS:  false,
  1165		Auth:   "localhost",
  1166	}
  1167	
  1168	// WriteDefaultConfigFile generates a new default high-level server configuration
  1169	// file at filePath. The default indexer will use SQLite.
  1170	// If filePath already exists, it is overwritten.
  1171	func WriteDefaultConfigFile(filePath string) error {
  1172		conf := defaultBaseConfig
  1173		blobDir, err := osutil.CamliBlobRoot()
  1174		if err != nil {
  1175			return err
  1176		}
  1177		varDir, err := osutil.CamliVarDir()
  1178		if err != nil {
  1179			return err
  1180		}
  1181		if err := wkfs.MkdirAll(blobDir, 0700); err != nil {
  1182			return fmt.Errorf("Could not create default blobs directory: %w", err)
  1183		}
  1184		conf.BlobPath = blobDir
  1185		conf.PackRelated = true
  1186	
  1187		conf.SQLite = filepath.Join(varDir, "index.sqlite")
  1188	
  1189		keyID, secretRing, err := getOrMakeKeyring()
  1190		if err != nil {
  1191			return err
  1192		}
  1193		conf.Identity = keyID
  1194		conf.IdentitySecretRing = secretRing
  1195	
  1196		confData, err := json.MarshalIndent(conf, "", "    ")
  1197		if err != nil {
  1198			return fmt.Errorf("Could not json encode config file: %w", err)
  1199		}
  1200	
  1201		if err := wkfs.WriteFile(filePath, confData, 0600); err != nil {
  1202			return fmt.Errorf("Could not create or write default server config: %w", err)
  1203		}
  1204	
  1205		return nil
  1206	}
  1207	
  1208	func getOrMakeKeyring() (keyID, secRing string, err error) {
  1209		secRing = osutil.SecretRingFile()
  1210		_, err = wkfs.Stat(secRing)
  1211		switch {
  1212		case err == nil:
  1213			keyID, err = jsonsign.KeyIdFromRing(secRing)
  1214			if err != nil {
  1215				err = fmt.Errorf("Could not find any keyID in file %q: %w", secRing, err)
  1216				return
  1217			}
  1218			log.Printf("Re-using identity with keyID %q found in file %s", keyID, secRing)
  1219		case os.IsNotExist(err):
  1220			keyID, err = jsonsign.GenerateNewSecRing(secRing)
  1221			if err != nil {
  1222				err = fmt.Errorf("Could not generate new secRing at file %q: %w", secRing, err)
  1223				return
  1224			}
  1225			log.Printf("Generated new identity with keyID %q in file %s", keyID, secRing)
  1226		default:
  1227			err = fmt.Errorf("Could not stat secret ring %q: %w", secRing, err)
  1228		}
  1229		return
  1230	}