1	/*
     2	Copyright 2012 The Perkeep Authors
     3	
     4	Licensed under the Apache License, Version 2.0 (the "License");
     5	you may not use this file except in compliance with the License.
     6	You may obtain a copy of the License at
     7	
     8	     http://www.apache.org/licenses/LICENSE-2.0
     9	
    10	Unless required by applicable law or agreed to in writing, software
    11	distributed under the License is distributed on an "AS IS" BASIS,
    12	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13	See the License for the specific language governing permissions and
    14	limitations under the License.
    15	*/
    16	
    17	package serverinit
    18	
    19	import (
    20		"encoding/json"
    21		"errors"
    22		"fmt"
    23		"log"
    24		"net/url"
    25		"os"
    26		"path"
    27		"path/filepath"
    28		"strconv"
    29		"strings"
    30	
    31		"go4.org/jsonconfig"
    32		"perkeep.org/internal/osutil"
    33		"perkeep.org/pkg/jsonsign"
    34		"perkeep.org/pkg/sorted"
    35		"perkeep.org/pkg/types/serverconfig"
    36	
    37		"go4.org/wkfs"
    38	)
    39	
    40	var (
    41		tempDir = os.TempDir
    42		noMkdir bool // for tests to not call os.Mkdir
    43	)
    44	
    45	type tlsOpts struct {
    46		autoCert  bool // use Perkeep's Let's Encrypt cache. but httpsCert takes precedence, if set.
    47		httpsCert string
    48		httpsKey  string
    49	}
    50	
    51	// genLowLevelConfig returns a low-level config from a high-level config.
    52	func genLowLevelConfig(conf *serverconfig.Config) (lowLevelConf *Config, err error) {
    53		b := &lowBuilder{
    54			high: conf,
    55			low: jsonconfig.Obj{
    56				"prefixes": make(map[string]interface{}),
    57			},
    58		}
    59		return b.build()
    60	}
    61	
    62	// A lowBuilder builds a low-level config from a high-level config.
    63	type lowBuilder struct {
    64		high *serverconfig.Config // high-level config (input)
    65		low  jsonconfig.Obj       // low-level handler config (output)
    66	}
    67	
    68	// args is an alias for map[string]interface{} just to cut down on
    69	// noise below.  But we take care to convert it back to
    70	// map[string]interface{} in the one place where we accept it.
    71	type args map[string]interface{}
    72	
    73	func (b *lowBuilder) addPrefix(at, handler string, a args) {
    74		v := map[string]interface{}{
    75			"handler": handler,
    76		}
    77		if a != nil {
    78			v["handlerArgs"] = (map[string]interface{})(a)
    79		}
    80		b.low["prefixes"].(map[string]interface{})[at] = v
    81	}
    82	
    83	func (b *lowBuilder) hasPrefix(p string) bool {
    84		_, ok := b.low["prefixes"].(map[string]interface{})[p]
    85		return ok
    86	}
    87	
    88	func (b *lowBuilder) runIndex() bool          { return b.high.RunIndex.Get() }
    89	func (b *lowBuilder) copyIndexToMemory() bool { return b.high.CopyIndexToMemory.Get() }
    90	
    91	type dbname string
    92	
    93	// possible arguments to dbName
    94	const (
    95		dbIndex           dbname = "index"
    96		dbBlobpackedIndex dbname = "blobpacked-index"
    97		dbDiskpackedIndex dbname = "diskpacked-index"
    98		dbUIThumbcache    dbname = "ui-thumbcache"
    99		dbSyncQueue       dbname = "queue-sync-to-" // only a prefix. the last part is the sync destination, e.g. "index".
   100	)
   101	
   102	// dbUnique returns the uniqueness string that is used in databases names to
   103	// differentiate them from databases used by other Perkeep instances on the same
   104	// DBMS.
   105	func (b *lowBuilder) dbUnique() string {
   106		if b.high.DBUnique != "" {
   107			return b.high.DBUnique
   108		}
   109		if b.high.Identity != "" {
   110			return strings.ToLower(b.high.Identity)
   111		}
   112		return osutil.Username() // may be empty, if $USER unset
   113	}
   114	
   115	// dbName returns which database to use for the provided user ("of"), which can
   116	// only be one of the const defined above. Returned values all follow the same name
   117	// scheme for consistency:
   118	// -prefixed with "pk_", so as to distinguish them from databases for other programs
   119	// -followed by a username-based uniqueness string
   120	// -last part says which component/part of perkeep it is about
   121	func (b *lowBuilder) dbName(of dbname) string {
   122		unique := b.dbUnique()
   123		if unique == "" {
   124			log.Printf("Could not define uniqueness for database of %q. Do not use the same index DBMS with other Perkeep instances.", of)
   125		}
   126		if unique == useDBNamesConfig {
   127			// this is the hint that we should revert to the old style DBNames, so this
   128			// instance can reuse its existing databases
   129			return b.oldDBNames(of)
   130		}
   131		prefix := "pk_"
   132		if unique != "" {
   133			prefix += unique + "_"
   134		}
   135		switch of {
   136		case dbIndex:
   137			if b.high.DBName != "" {
   138				return b.high.DBName
   139			}
   140			return prefix + "index"
   141		case dbBlobpackedIndex:
   142			return prefix + "blobpacked"
   143		case dbDiskpackedIndex:
   144			return prefix + "diskpacked"
   145		case dbUIThumbcache:
   146			return prefix + "uithumbmeta"
   147		}
   148		asString := string(of)
   149		if strings.HasPrefix(asString, string(dbSyncQueue)) {
   150			return prefix + "syncto_" + strings.TrimPrefix(asString, string(dbSyncQueue))
   151		}
   152		return ""
   153	}
   154	
   155	// As of rev 7eda9fd5027fda88166d6c03b6490cffbf2de5fb, we changed how the
   156	// databases names were defined. But we wanted the existing GCE instances to keep
   157	// on working with the old names, so that nothing would break for existing users,
   158	// without any intervention needed. Through the help of the perkeep-config-version
   159	// variable, set by the GCE launcher, we can know whether an instance is such an
   160	// "old" one, and in that case we keep on using the old database names. oldDBNames
   161	// returns these names.
   162	func (b *lowBuilder) oldDBNames(of dbname) string {
   163		switch of {
   164		case dbIndex:
   165			return "camlistore_index"
   166		case dbBlobpackedIndex:
   167			return "blobpacked_index"
   168		case "queue-sync-to-index":
   169			return "sync_index_queue"
   170		case dbUIThumbcache:
   171			return "ui_thumbmeta_cache"
   172		}
   173		return ""
   174	}
   175	
   176	var errNoOwner = errors.New("no owner")
   177	
   178	// Error is errNoOwner if no identity configured
   179	func (b *lowBuilder) searchOwner() (owner *serverconfig.Owner, err error) {
   180		if b.high.Identity == "" {
   181			return nil, errNoOwner
   182		}
   183		if b.high.IdentitySecretRing == "" {
   184			return nil, errNoOwner
   185		}
   186		return &serverconfig.Owner{
   187			Identity:    b.high.Identity,
   188			SecringFile: b.high.IdentitySecretRing,
   189		}, nil
   190	}
   191	
   192	// longIdentity returns the long form (16 chars) of the GPG key ID, in case the
   193	// user provided the short form (8 chars) or the fingerprint (40 chars) in the config.
   194	func (b *lowBuilder) longIdentity() (string, error) {
   195		if b.high.Identity == "" {
   196			return "", errNoOwner
   197		}
   198		if strings.ToUpper(b.high.Identity) != b.high.Identity {
   199			return "", fmt.Errorf("identity %q is not all upper-case", b.high.Identity)
   200		}
   201		if len(b.high.Identity) == 16 {
   202			return b.high.Identity, nil
   203		}
   204		if len(b.high.Identity) == 40 {
   205			return b.high.Identity[24:], nil
   206		}
   207		if b.high.IdentitySecretRing == "" {
   208			return "", errNoOwner
   209		}
   210		keyID, err := jsonsign.KeyIdFromRing(b.high.IdentitySecretRing)
   211		if err != nil {
   212			return "", fmt.Errorf("could not find any keyID in file %q: %v", b.high.IdentitySecretRing, err)
   213		}
   214		if !strings.HasSuffix(keyID, b.high.Identity) {
   215			return "", fmt.Errorf("%q identity not found in secret ring %v", b.high.Identity, b.high.IdentitySecretRing)
   216		}
   217		return keyID, nil
   218	}
   219	
   220	func addAppConfig(config map[string]interface{}, appConfig *serverconfig.App, low jsonconfig.Obj) {
   221		if appConfig.Listen != "" {
   222			config["listen"] = appConfig.Listen
   223		}
   224		if appConfig.APIHost != "" {
   225			config["apiHost"] = appConfig.APIHost
   226		}
   227		if appConfig.BackendURL != "" {
   228			config["backendURL"] = appConfig.BackendURL
   229		}
   230		if low["listen"] != nil && low["listen"].(string) != "" {
   231			config["serverListen"] = low["listen"].(string)
   232		}
   233		if low["baseURL"] != nil && low["baseURL"].(string) != "" {
   234			config["serverBaseURL"] = low["baseURL"].(string)
   235		}
   236	}
   237	
   238	func (b *lowBuilder) addPublishedConfig(tlsO *tlsOpts) error {
   239		published := b.high.Publish
   240		for k, v := range published {
   241			// trick in case all of the fields of v.App were omitted, which would leave v.App nil.
   242			if v.App == nil {
   243				v.App = &serverconfig.App{}
   244			}
   245			if v.CamliRoot == "" {
   246				return fmt.Errorf("missing \"camliRoot\" key in configuration for %s", k)
   247			}
   248			if v.GoTemplate == "" {
   249				return fmt.Errorf("missing \"goTemplate\" key in configuration for %s", k)
   250			}
   251			appConfig := map[string]interface{}{
   252				"camliRoot":  v.CamliRoot,
   253				"cacheRoot":  v.CacheRoot,
   254				"goTemplate": v.GoTemplate,
   255			}
   256			if v.SourceRoot != "" {
   257				appConfig["sourceRoot"] = v.SourceRoot
   258			}
   259			if v.HTTPSCert != "" && v.HTTPSKey != "" {
   260				// user can specify these directly in the publish section
   261				appConfig["httpsCert"] = v.HTTPSCert
   262				appConfig["httpsKey"] = v.HTTPSKey
   263			} else {
   264				// default to Perkeep parameters, if any
   265				if tlsO != nil {
   266					if tlsO.autoCert {
   267						appConfig["certManager"] = tlsO.autoCert
   268					}
   269					if tlsO.httpsCert != "" {
   270						appConfig["httpsCert"] = tlsO.httpsCert
   271					}
   272					if tlsO.httpsKey != "" {
   273						appConfig["httpsKey"] = tlsO.httpsKey
   274					}
   275				}
   276			}
   277			program := "publisher"
   278			if v.Program != "" {
   279				program = v.Program
   280			}
   281			a := args{
   282				"prefix":    k,
   283				"program":   program,
   284				"appConfig": appConfig,
   285			}
   286			addAppConfig(a, v.App, b.low)
   287			b.addPrefix(k, "app", a)
   288		}
   289		return nil
   290	}
   291	
   292	func (b *lowBuilder) addScanCabConfig(tlsO *tlsOpts) error {
   293		if b.high.ScanCab == nil {
   294			return nil
   295		}
   296		scancab := b.high.ScanCab
   297		if scancab.App == nil {
   298			scancab.App = &serverconfig.App{}
   299		}
   300		if scancab.Prefix == "" {
   301			return errors.New("missing \"prefix\" key in configuration for scanning cabinet")
   302		}
   303	
   304		program := "scanningcabinet"
   305		if scancab.Program != "" {
   306			program = scancab.Program
   307		}
   308	
   309		auth := scancab.Auth
   310		if auth == "" {
   311			auth = b.high.Auth
   312		}
   313		appConfig := map[string]interface{}{
   314			"auth": auth,
   315		}
   316		if scancab.HTTPSCert != "" && scancab.HTTPSKey != "" {
   317			appConfig["httpsCert"] = scancab.HTTPSCert
   318			appConfig["httpsKey"] = scancab.HTTPSKey
   319		} else {
   320			// default to Perkeep parameters, if any
   321			if tlsO != nil {
   322				appConfig["httpsCert"] = tlsO.httpsCert
   323				appConfig["httpsKey"] = tlsO.httpsKey
   324			}
   325		}
   326		a := args{
   327			"prefix":    scancab.Prefix,
   328			"program":   program,
   329			"appConfig": appConfig,
   330		}
   331		addAppConfig(a, scancab.App, b.low)
   332		b.addPrefix(scancab.Prefix, "app", a)
   333		return nil
   334	}
   335	
   336	func (b *lowBuilder) sortedName() string {
   337		switch {
   338		case b.high.MySQL != "":
   339			return "MySQL"
   340		case b.high.PostgreSQL != "":
   341			return "PostgreSQL"
   342		case b.high.Mongo != "":
   343			return "MongoDB"
   344		case b.high.MemoryIndex:
   345			return "in memory LevelDB"
   346		case b.high.SQLite != "":
   347			return "SQLite"
   348		case b.high.KVFile != "":
   349			return "KVFile"
   350		case b.high.LevelDB != "":
   351			return "LevelDB"
   352		}
   353		panic("internal error: sortedName didn't find a sorted implementation")
   354	}
   355	
   356	// kvFileType returns the file based sorted type defined for index storage, if
   357	// any. It defaults to "leveldb" otherwise.
   358	func (b *lowBuilder) kvFileType() string {
   359		switch {
   360		case b.high.SQLite != "":
   361			return "sqlite"
   362		case b.high.KVFile != "":
   363			return "kv"
   364		case b.high.LevelDB != "":
   365			return "leveldb"
   366		default:
   367			return sorted.DefaultKVFileType
   368		}
   369	}
   370	
   371	func (b *lowBuilder) addUIConfig() {
   372		args := map[string]interface{}{
   373			"cache": "/cache/",
   374		}
   375		if b.high.SourceRoot != "" {
   376			args["sourceRoot"] = b.high.SourceRoot
   377		}
   378		var thumbCache map[string]interface{}
   379		if b.high.BlobPath != "" {
   380			thumbCache = map[string]interface{}{
   381				"type": b.kvFileType(),
   382				"file": filepath.Join(b.high.BlobPath, "thumbmeta."+b.kvFileType()),
   383			}
   384		}
   385		if thumbCache == nil {
   386			sorted, err := b.sortedStorage(dbUIThumbcache)
   387			if err == nil {
   388				thumbCache = sorted
   389			}
   390		}
   391		if thumbCache != nil {
   392			args["scaledImage"] = thumbCache
   393		}
   394		b.addPrefix("/ui/", "ui", args)
   395	}
   396	
   397	func (b *lowBuilder) mongoIndexStorage(confStr string, sortedType dbname) (map[string]interface{}, error) {
   398		dbName := b.dbName(sortedType)
   399		if dbName == "" {
   400			return nil, fmt.Errorf("no database name configured for sorted store %q", sortedType)
   401		}
   402		fields := strings.Split(confStr, "@")
   403		if len(fields) == 2 {
   404			host := fields[1]
   405			fields = strings.Split(fields[0], ":")
   406			if len(fields) == 2 {
   407				user, pass := fields[0], fields[1]
   408				return map[string]interface{}{
   409					"type":     "mongo",
   410					"host":     host,
   411					"user":     user,
   412					"password": pass,
   413					"database": dbName,
   414				}, nil
   415			}
   416		}
   417		return nil, errors.New("Malformed mongo config string; want form: \"user:password@host\"")
   418	}
   419	
   420	// parses "user@host:password", which you think would be easy, but we
   421	// documented this format without thinking about port numbers, so this
   422	// uses heuristics to guess what extra colons mean.
   423	func parseUserHostPass(v string) (user, host, password string, ok bool) {
   424		f := strings.SplitN(v, "@", 2)
   425		if len(f) != 2 {
   426			return
   427		}
   428		user = f[0]
   429		f = strings.Split(f[1], ":")
   430		if len(f) < 2 {
   431			return "", "", "", false
   432		}
   433		host = f[0]
   434		f = f[1:]
   435		if len(f) >= 2 {
   436			if _, err := strconv.ParseUint(f[0], 10, 16); err == nil {
   437				host = host + ":" + f[0]
   438				f = f[1:]
   439			}
   440		}
   441		password = strings.Join(f, ":")
   442		ok = true
   443		return
   444	}
   445	
   446	func (b *lowBuilder) dbIndexStorage(rdbms, confStr string, sortedType dbname) (map[string]interface{}, error) {
   447		dbName := b.dbName(sortedType)
   448		if dbName == "" {
   449			return nil, fmt.Errorf("no database name configured for sorted store %q", sortedType)
   450		}
   451		user, host, password, ok := parseUserHostPass(confStr)
   452		if !ok {
   453			return nil, fmt.Errorf("Malformed %s config string. Want: \"user@host:password\"", rdbms)
   454		}
   455		return map[string]interface{}{
   456			"type":     rdbms,
   457			"host":     host,
   458			"user":     user,
   459			"password": password,
   460			"database": dbName,
   461		}, nil
   462	}
   463	
   464	func (b *lowBuilder) sortedStorage(sortedType dbname) (map[string]interface{}, error) {
   465		return b.sortedStorageAt(sortedType, "")
   466	}
   467	
   468	// sortedDBMS returns the configuration for a name database on one of the
   469	// DBMS, if any was found in the configuration. It returns nil otherwise.
   470	func (b *lowBuilder) sortedDBMS(named dbname) (map[string]interface{}, error) {
   471		if b.high.MySQL != "" {
   472			return b.dbIndexStorage("mysql", b.high.MySQL, named)
   473		}
   474		if b.high.PostgreSQL != "" {
   475			return b.dbIndexStorage("postgres", b.high.PostgreSQL, named)
   476		}
   477		if b.high.Mongo != "" {
   478			return b.mongoIndexStorage(b.high.Mongo, named)
   479		}
   480		return nil, nil
   481	}
   482	
   483	// filePrefix gives a file path of where to put the database. It can be omitted by
   484	// some sorted implementations, but is required by others.
   485	// The filePrefix should be to a file, not a directory, and should not end in a ".ext" extension.
   486	// An extension like ".kv" or ".sqlite" will be added.
   487	func (b *lowBuilder) sortedStorageAt(sortedType dbname, filePrefix string) (map[string]interface{}, error) {
   488		dbms, err := b.sortedDBMS(sortedType)
   489		if err != nil {
   490			return nil, err
   491		}
   492		if dbms != nil {
   493			return dbms, nil
   494		}
   495		if b.high.MemoryIndex {
   496			return map[string]interface{}{
   497				"type": "memory",
   498			}, nil
   499		}
   500		if sortedType != "index" && filePrefix == "" {
   501			return nil, fmt.Errorf("internal error: use of sortedStorageAt with a non-index type (%v) and no file location for non-database sorted implementation", sortedType)
   502		}
   503		// dbFile returns path directly if sortedType == "index", else it returns filePrefix+"."+ext.
   504		dbFile := func(path, ext string) string {
   505			if sortedType == "index" {
   506				return path
   507			}
   508			return filePrefix + "." + ext
   509		}
   510		if b.high.SQLite != "" {
   511			return map[string]interface{}{
   512				"type": "sqlite",
   513				"file": dbFile(b.high.SQLite, "sqlite"),
   514			}, nil
   515		}
   516		if b.high.KVFile != "" {
   517			return map[string]interface{}{
   518				"type": "kv",
   519				"file": dbFile(b.high.KVFile, "kv"),
   520			}, nil
   521		}
   522		if b.high.LevelDB != "" {
   523			return map[string]interface{}{
   524				"type": "leveldb",
   525				"file": dbFile(b.high.LevelDB, "leveldb"),
   526			}, nil
   527		}
   528		panic("internal error: sortedStorageAt didn't find a sorted implementation")
   529	}
   530	
   531	func (b *lowBuilder) thatQueueUnlessMemory(thatQueue map[string]interface{}) (queue map[string]interface{}) {
   532		// TODO(mpl): what about if b.high.MemoryIndex ?
   533		if b.high.MemoryStorage {
   534			return map[string]interface{}{
   535				"type": "memory",
   536			}
   537		}
   538		return thatQueue
   539	}
   540	
   541	func (b *lowBuilder) addS3Config(s3 string, vendor string) error {
   542		f := strings.SplitN(s3, ":", 4)
   543		if len(f) < 3 {
   544			m := fmt.Sprintf(`genconfig: expected "%s" field to be of form "access_key_id:secret_access_key:bucket[/optional/dir][:hostname]"`, vendor)
   545			return errors.New(m)
   546		}
   547		accessKey, secret, bucket := f[0], f[1], f[2]
   548		var hostname string
   549		if len(f) == 4 {
   550			hostname = f[3]
   551		}
   552		isReplica := b.hasPrefix("/bs/")
   553		s3Prefix := "/bs/"
   554		if isReplica {
   555			s3Prefix = fmt.Sprintf("/sto-%s/", vendor)
   556		}
   557	
   558		s3Args := func(bucket string) args {
   559			a := args{
   560				"bucket":                bucket,
   561				"aws_access_key":        accessKey,
   562				"aws_secret_access_key": secret,
   563			}
   564			if hostname != "" {
   565				a["hostname"] = hostname
   566			}
   567			return a
   568		}
   569	
   570		if !b.high.PackRelated {
   571			b.addPrefix(s3Prefix, "storage-s3", s3Args(bucket))
   572		} else {
   573			bsLoose := "/bs-loose/"
   574			bsPacked := "/bs-packed/"
   575			if isReplica {
   576				bsLoose = fmt.Sprintf("/sto-%s-bs-loose/", vendor)
   577				bsPacked = fmt.Sprintf("/sto-%s-bs-packed/", vendor)
   578			}
   579	
   580			b.addPrefix(bsLoose, "storage-s3", s3Args(path.Join(bucket, "loose")))
   581			b.addPrefix(bsPacked, "storage-s3", s3Args(path.Join(bucket, "packed")))
   582	
   583			// If index is DBMS, then blobPackedIndex is in DBMS too.
   584			// Otherwise blobPackedIndex is same file-based DB as the index,
   585			// in same dir, but named packindex.dbtype.
   586			blobPackedIndex, err := b.sortedStorageAt(dbBlobpackedIndex, filepath.Join(b.indexFileDir(), "packindex"))
   587			if err != nil {
   588				return err
   589			}
   590			b.addPrefix(s3Prefix, "storage-blobpacked", args{
   591				"smallBlobs": bsLoose,
   592				"largeBlobs": bsPacked,
   593				"metaIndex":  blobPackedIndex,
   594			})
   595		}
   596	
   597		if isReplica {
   598			if b.high.BlobPath == "" && !b.high.MemoryStorage {
   599				panic("unexpected empty blobpath with sync-to-s3")
   600			}
   601			p := fmt.Sprintf("/sync-to-%s/", vendor)
   602			queue := fmt.Sprintf("sync-to-%s-queue.", vendor)
   603			b.addPrefix(p, "sync", args{
   604				"from": "/bs/",
   605				"to":   s3Prefix,
   606				"queue": b.thatQueueUnlessMemory(
   607					map[string]interface{}{
   608						"type": b.kvFileType(),
   609						"file": filepath.Join(b.high.BlobPath, queue+b.kvFileType()),
   610					}),
   611			})
   612			return nil
   613		}
   614	
   615		// TODO(mpl): s3CacheBucket
   616		// See https://perkeep.org/issue/85
   617		b.addPrefix("/cache/", "storage-filesystem", args{
   618			"path": filepath.Join(tempDir(), "camli-cache"),
   619		})
   620	
   621		return nil
   622	}
   623	
   624	func (b *lowBuilder) addB2Config(b2 string) error {
   625		return b.addS3Config(b2, "b2")
   626	}
   627	
   628	func (b *lowBuilder) addGoogleDriveConfig(v string) error {
   629		f := strings.SplitN(v, ":", 4)
   630		if len(f) != 4 {
   631			return errors.New(`genconfig: expected "googledrive" field to be of form "client_id:client_secret:refresh_token:parent_id"`)
   632		}
   633		clientId, secret, refreshToken, parentId := f[0], f[1], f[2], f[3]
   634	
   635		isPrimary := !b.hasPrefix("/bs/")
   636		prefix := ""
   637		if isPrimary {
   638			prefix = "/bs/"
   639			if b.high.PackRelated {
   640				return errors.New("TODO: finish packRelated support for Google Drive")
   641			}
   642		} else {
   643			prefix = "/sto-googledrive/"
   644		}
   645		b.addPrefix(prefix, "storage-googledrive", args{
   646			"parent_id": parentId,
   647			"auth": map[string]interface{}{
   648				"client_id":     clientId,
   649				"client_secret": secret,
   650				"refresh_token": refreshToken,
   651			},
   652		})
   653	
   654		if isPrimary {
   655			b.addPrefix("/cache/", "storage-filesystem", args{
   656				"path": filepath.Join(tempDir(), "camli-cache"),
   657			})
   658		} else {
   659			b.addPrefix("/sync-to-googledrive/", "sync", args{
   660				"from": "/bs/",
   661				"to":   prefix,
   662				"queue": b.thatQueueUnlessMemory(
   663					map[string]interface{}{
   664						"type": b.kvFileType(),
   665						"file": filepath.Join(b.high.BlobPath, "sync-to-googledrive-queue."+b.kvFileType()),
   666					}),
   667			})
   668		}
   669	
   670		return nil
   671	}
   672	
   673	var errGCSUsage = errors.New(`genconfig: expected "googlecloudstorage" field to be of form "client_id:client_secret:refresh_token:bucket[/dir/]" or ":bucketname[/dir/]"`)
   674	
   675	func (b *lowBuilder) addGoogleCloudStorageConfig(v string) error {
   676		var clientID, secret, refreshToken, bucket string
   677		f := strings.SplitN(v, ":", 4)
   678		switch len(f) {
   679		default:
   680			return errGCSUsage
   681		case 4:
   682			clientID, secret, refreshToken, bucket = f[0], f[1], f[2], f[3]
   683		case 2:
   684			if f[0] != "" {
   685				return errGCSUsage
   686			}
   687			bucket = f[1]
   688			clientID = "auto"
   689		}
   690	
   691		isReplica := b.hasPrefix("/bs/")
   692		gsPrefix := "/bs/"
   693		if isReplica {
   694			gsPrefix = "/sto-googlecloudstorage/"
   695		}
   696	
   697		gsArgs := func(bucket string) args {
   698			a := args{
   699				"bucket": bucket,
   700				"auth": map[string]interface{}{
   701					"client_id":     clientID,
   702					"client_secret": secret,
   703					"refresh_token": refreshToken,
   704				},
   705			}
   706			return a
   707		}
   708	
   709		if !b.high.PackRelated {
   710			b.addPrefix(gsPrefix, "storage-googlecloudstorage", gsArgs(bucket))
   711		} else {
   712			bsLoose := "/bs-loose/"
   713			bsPacked := "/bs-packed/"
   714			if isReplica {
   715				bsLoose = "/sto-googlecloudstorage-bs-loose/"
   716				bsPacked = "/sto-googlecloudstorage-bs-packed/"
   717			}
   718	
   719			b.addPrefix(bsLoose, "storage-googlecloudstorage", gsArgs(path.Join(bucket, "loose")))
   720			b.addPrefix(bsPacked, "storage-googlecloudstorage", gsArgs(path.Join(bucket, "packed")))
   721	
   722			// If index is DBMS, then blobPackedIndex is in DBMS too.
   723			// Otherwise blobPackedIndex is same file-based DB as the index,
   724			// in same dir, but named packindex.dbtype.
   725			blobPackedIndex, err := b.sortedStorageAt(dbBlobpackedIndex, filepath.Join(b.indexFileDir(), "packindex"))
   726			if err != nil {
   727				return err
   728			}
   729			b.addPrefix(gsPrefix, "storage-blobpacked", args{
   730				"smallBlobs": bsLoose,
   731				"largeBlobs": bsPacked,
   732				"metaIndex":  blobPackedIndex,
   733			})
   734		}
   735	
   736		if isReplica {
   737			if b.high.BlobPath == "" && !b.high.MemoryStorage {
   738				panic("unexpected empty blobpath with sync-to-googlecloudstorage")
   739			}
   740			b.addPrefix("/sync-to-googlecloudstorage/", "sync", args{
   741				"from": "/bs/",
   742				"to":   gsPrefix,
   743				"queue": b.thatQueueUnlessMemory(
   744					map[string]interface{}{
   745						"type": b.kvFileType(),
   746						"file": filepath.Join(b.high.BlobPath, "sync-to-googlecloud-queue."+b.kvFileType()),
   747					}),
   748			})
   749			return nil
   750		}
   751	
   752		// TODO: cacheBucket like s3CacheBucket?
   753		b.addPrefix("/cache/", "storage-filesystem", args{
   754			"path": filepath.Join(tempDir(), "camli-cache"),
   755		})
   756	
   757		return nil
   758	}
   759	
   760	// indexFileDir returns the directory of the sqlite or kv file, or the
   761	// empty string.
   762	func (b *lowBuilder) indexFileDir() string {
   763		switch {
   764		case b.high.SQLite != "":
   765			return filepath.Dir(b.high.SQLite)
   766		case b.high.KVFile != "":
   767			return filepath.Dir(b.high.KVFile)
   768		case b.high.LevelDB != "":
   769			return filepath.Dir(b.high.LevelDB)
   770		}
   771		return ""
   772	}
   773	
   774	func (b *lowBuilder) syncToIndexArgs() (map[string]interface{}, error) {
   775		a := map[string]interface{}{
   776			"from": "/bs/",
   777			"to":   "/index/",
   778		}
   779	
   780		// TODO(mpl): see if we want to have the same logic with all the other queues. probably.
   781		const sortedType = "queue-sync-to-index"
   782		if dbName := b.dbName(sortedType); dbName != "" {
   783			qj, err := b.sortedDBMS(sortedType)
   784			if err != nil {
   785				return nil, err
   786			}
   787			if qj == nil && b.high.MemoryIndex {
   788				qj = map[string]interface{}{
   789					"type": "memory",
   790				}
   791			}
   792			if qj != nil {
   793				// i.e. the index is configured on a DBMS, so we put the queue there too
   794				a["queue"] = qj
   795				return a, nil
   796			}
   797		}
   798	
   799		// TODO: currently when using s3, the index must be
   800		// sqlite or kvfile, since only through one of those
   801		// can we get a directory.
   802		if !b.high.MemoryStorage && b.high.BlobPath == "" && b.indexFileDir() == "" {
   803			// We don't actually have a working sync handler, but we keep a stub registered
   804			// so it can be referred to from other places.
   805			// See http://perkeep.org/issue/201
   806			a["idle"] = true
   807			return a, nil
   808		}
   809	
   810		dir := b.high.BlobPath
   811		if dir == "" {
   812			dir = b.indexFileDir()
   813		}
   814		a["queue"] = b.thatQueueUnlessMemory(
   815			map[string]interface{}{
   816				"type": b.kvFileType(),
   817				"file": filepath.Join(dir, "sync-to-index-queue."+b.kvFileType()),
   818			})
   819	
   820		return a, nil
   821	}
   822	
   823	func (b *lowBuilder) genLowLevelPrefixes() error {
   824		root := "/bs/"
   825		pubKeyDest := root
   826		if b.runIndex() {
   827			root = "/bs-and-maybe-also-index/"
   828			pubKeyDest = "/bs-and-index/"
   829		}
   830	
   831		rootArgs := map[string]interface{}{
   832			"stealth":      false,
   833			"blobRoot":     root,
   834			"helpRoot":     "/help/",
   835			"statusRoot":   "/status/",
   836			"jsonSignRoot": "/sighelper/",
   837		}
   838		if b.high.OwnerName != "" {
   839			rootArgs["ownerName"] = b.high.OwnerName
   840		}
   841		if b.runIndex() {
   842			rootArgs["searchRoot"] = "/my-search/"
   843		}
   844		if path := b.high.ShareHandlerPath; path != "" {
   845			rootArgs["shareRoot"] = path
   846			b.addPrefix(path, "share", args{
   847				"blobRoot": "/bs/",
   848				"index":    "/index/",
   849			})
   850		}
   851		b.addPrefix("/", "root", rootArgs)
   852		b.addPrefix("/setup/", "setup", nil)
   853		b.addPrefix("/status/", "status", nil)
   854		b.addPrefix("/help/", "help", nil)
   855	
   856		importerArgs := args{}
   857		if b.high.Flickr != "" {
   858			importerArgs["flickr"] = map[string]interface{}{
   859				"clientSecret": b.high.Flickr,
   860			}
   861		}
   862		if b.high.Picasa != "" {
   863			importerArgs["picasa"] = map[string]interface{}{
   864				"clientSecret": b.high.Picasa,
   865			}
   866		}
   867		if b.high.Instapaper != "" {
   868			importerArgs["instapaper"] = map[string]interface{}{
   869				"clientSecret": b.high.Instapaper,
   870			}
   871		}
   872		if b.runIndex() {
   873			b.addPrefix("/importer/", "importer", importerArgs)
   874		}
   875	
   876		b.addPrefix("/sighelper/", "jsonsign", args{
   877			"secretRing":    b.high.IdentitySecretRing,
   878			"keyId":         b.high.Identity,
   879			"publicKeyDest": pubKeyDest,
   880		})
   881	
   882		storageType := "filesystem"
   883		if b.high.PackBlobs {
   884			storageType = "diskpacked"
   885		}
   886		if b.high.BlobPath != "" {
   887			if b.high.PackRelated {
   888				b.addPrefix("/bs-loose/", "storage-filesystem", args{
   889					"path": b.high.BlobPath,
   890				})
   891				b.addPrefix("/bs-packed/", "storage-filesystem", args{
   892					"path": filepath.Join(b.high.BlobPath, "packed"),
   893				})
   894				blobPackedIndex, err := b.sortedStorageAt(dbBlobpackedIndex, filepath.Join(b.high.BlobPath, "packed", "packindex"))
   895				if err != nil {
   896					return err
   897				}
   898				b.addPrefix("/bs/", "storage-blobpacked", args{
   899					"smallBlobs": "/bs-loose/",
   900					"largeBlobs": "/bs-packed/",
   901					"metaIndex":  blobPackedIndex,
   902				})
   903			} else if b.high.PackBlobs {
   904				diskpackedIndex, err := b.sortedStorageAt(dbDiskpackedIndex, filepath.Join(b.high.BlobPath, "diskpacked-index"))
   905				if err != nil {
   906					return err
   907				}
   908				b.addPrefix("/bs/", "storage-"+storageType, args{
   909					"path":      b.high.BlobPath,
   910					"metaIndex": diskpackedIndex,
   911				})
   912			} else {
   913				b.addPrefix("/bs/", "storage-"+storageType, args{
   914					"path": b.high.BlobPath,
   915				})
   916			}
   917			if b.high.PackBlobs {
   918				b.addPrefix("/cache/", "storage-"+storageType, args{
   919					"path": filepath.Join(b.high.BlobPath, "/cache"),
   920					"metaIndex": map[string]interface{}{
   921						"type": b.kvFileType(),
   922						"file": filepath.Join(b.high.BlobPath, "cache", "index."+b.kvFileType()),
   923					},
   924				})
   925			} else {
   926				b.addPrefix("/cache/", "storage-"+storageType, args{
   927					"path": filepath.Join(b.high.BlobPath, "/cache"),
   928				})
   929			}
   930		} else if b.high.MemoryStorage {
   931			b.addPrefix("/bs/", "storage-memory", nil)
   932			b.addPrefix("/cache/", "storage-memory", nil)
   933		}
   934	
   935		if b.runIndex() {
   936			syncArgs, err := b.syncToIndexArgs()
   937			if err != nil {
   938				return err
   939			}
   940			b.addPrefix("/sync/", "sync", syncArgs)
   941	
   942			b.addPrefix("/bs-and-index/", "storage-replica", args{
   943				"backends": []interface{}{"/bs/", "/index/"},
   944			})
   945	
   946			b.addPrefix("/bs-and-maybe-also-index/", "storage-cond", args{
   947				"write": map[string]interface{}{
   948					"if":   "isSchema",
   949					"then": "/bs-and-index/",
   950					"else": "/bs/",
   951				},
   952				"read": "/bs/",
   953			})
   954	
   955			owner, err := b.searchOwner()
   956			if err != nil {
   957				return err
   958			}
   959			searchArgs := args{
   960				"index": "/index/",
   961				"owner": map[string]interface{}{
   962					"identity":    owner.Identity,
   963					"secringFile": owner.SecringFile,
   964				},
   965			}
   966			if b.copyIndexToMemory() {
   967				searchArgs["slurpToMemory"] = true
   968			}
   969			b.addPrefix("/my-search/", "search", searchArgs)
   970		}
   971	
   972		return nil
   973	}
   974	
   975	func (b *lowBuilder) build() (*Config, error) {
   976		conf, low := b.high, b.low
   977		if conf.HTTPS {
   978			if (conf.HTTPSCert != "") != (conf.HTTPSKey != "") {
   979				return nil, errors.New("Must set both httpsCert and httpsKey (or neither to generate a self-signed cert)")
   980			}
   981			if conf.HTTPSCert != "" {
   982				low["httpsCert"] = conf.HTTPSCert
   983				low["httpsKey"] = conf.HTTPSKey
   984			}
   985		}
   986	
   987		if conf.BaseURL != "" {
   988			u, err := url.Parse(conf.BaseURL)
   989			if err != nil {
   990				return nil, fmt.Errorf("Error parsing baseURL %q as a URL: %v", conf.BaseURL, err)
   991			}
   992			if u.Path != "" && u.Path != "/" {
   993				return nil, fmt.Errorf("baseURL can't have a path, only a scheme, host, and optional port")
   994			}
   995			u.Path = ""
   996			low["baseURL"] = u.String()
   997		}
   998		if conf.Listen != "" {
   999			low["listen"] = conf.Listen
  1000		}
  1001		if conf.PackBlobs && conf.PackRelated {
  1002			return nil, errors.New("can't use both packBlobs (for 'diskpacked') and packRelated (for 'blobpacked')")
  1003		}
  1004		low["https"] = conf.HTTPS
  1005		low["auth"] = conf.Auth
  1006	
  1007		numIndexers := numSet(conf.LevelDB, conf.Mongo, conf.MySQL, conf.PostgreSQL, conf.SQLite, conf.KVFile, conf.MemoryIndex)
  1008	
  1009		switch {
  1010		case b.runIndex() && numIndexers == 0:
  1011			return nil, fmt.Errorf("Unless runIndex is set to false, you must specify an index option (kvIndexFile, leveldb, mongo, mysql, postgres, sqlite, memoryIndex).")
  1012		case b.runIndex() && numIndexers != 1:
  1013			return nil, fmt.Errorf("With runIndex set true, you can only pick exactly one indexer (mongo, mysql, postgres, sqlite, kvIndexFile, leveldb, memoryIndex).")
  1014		case !b.runIndex() && numIndexers != 0:
  1015			log.Printf("Indexer disabled, but %v will be used for other indexes, queues, caches, etc.", b.sortedName())
  1016		}
  1017	
  1018		longID, err := b.longIdentity()
  1019		if err != nil {
  1020			return nil, err
  1021		}
  1022		b.high.Identity = longID
  1023	
  1024		noLocalDisk := conf.BlobPath == ""
  1025		if noLocalDisk {
  1026			if !conf.MemoryStorage && conf.S3 == "" && conf.B2 == "" && conf.GoogleCloudStorage == "" {
  1027				return nil, errors.New("Unless memoryStorage is set, you must specify at least one storage option for your blobserver (blobPath (for localdisk), s3, b2, googlecloudstorage).")
  1028			}
  1029			if !conf.MemoryStorage && conf.S3 != "" && conf.GoogleCloudStorage != "" {
  1030				return nil, errors.New("Using S3 as a primary storage and Google Cloud Storage as a mirror is not supported for now.")
  1031			}
  1032			if !conf.MemoryStorage && conf.B2 != "" && conf.GoogleCloudStorage != "" {
  1033				return nil, errors.New("Using B2 as a primary storage and Google Cloud Storage as a mirror is not supported for now.")
  1034			}
  1035		}
  1036		if conf.ShareHandler && conf.ShareHandlerPath == "" {
  1037			conf.ShareHandlerPath = "/share/"
  1038		}
  1039		if conf.MemoryStorage {
  1040			noMkdir = true
  1041			if conf.BlobPath != "" {
  1042				return nil, errors.New("memoryStorage and blobPath are mutually exclusive.")
  1043			}
  1044			if conf.PackRelated {
  1045				return nil, errors.New("memoryStorage doesn't support packRelated.")
  1046			}
  1047		}
  1048	
  1049		if err := b.genLowLevelPrefixes(); err != nil {
  1050			return nil, err
  1051		}
  1052	
  1053		var cacheDir string
  1054		if noLocalDisk {
  1055			// Whether perkeepd is run from EC2 or not, we use
  1056			// a temp dir as the cache when primary storage is S3.
  1057			// TODO(mpl): s3CacheBucket
  1058			// See https://perkeep.org/issue/85
  1059			cacheDir = filepath.Join(tempDir(), "camli-cache")
  1060		} else {
  1061			cacheDir = filepath.Join(conf.BlobPath, "cache")
  1062		}
  1063		if !noMkdir {
  1064			if err := os.MkdirAll(cacheDir, 0700); err != nil {
  1065				return nil, fmt.Errorf("Could not create blobs cache dir %s: %v", cacheDir, err)
  1066			}
  1067		}
  1068	
  1069		if len(conf.Publish) > 0 {
  1070			if !b.runIndex() {
  1071				return nil, fmt.Errorf("publishing requires an index")
  1072			}
  1073			var tlsO *tlsOpts
  1074			httpsCert, ok1 := low["httpsCert"].(string)
  1075			httpsKey, ok2 := low["httpsKey"].(string)
  1076			if ok1 && ok2 {
  1077				tlsO = &tlsOpts{
  1078					httpsCert: httpsCert,
  1079					httpsKey:  httpsKey,
  1080				}
  1081			} else if conf.HTTPS {
  1082				tlsO = &tlsOpts{
  1083					autoCert: true,
  1084				}
  1085			}
  1086			if err := b.addPublishedConfig(tlsO); err != nil {
  1087				return nil, fmt.Errorf("Could not generate config for published: %v", err)
  1088			}
  1089		}
  1090	
  1091		if conf.ScanCab != nil {
  1092			if !b.runIndex() {
  1093				return nil, fmt.Errorf("scanning cabinet requires an index")
  1094			}
  1095			var tlsO *tlsOpts
  1096			httpsCert, ok1 := low["httpsCert"].(string)
  1097			httpsKey, ok2 := low["httpsKey"].(string)
  1098			if ok1 && ok2 {
  1099				tlsO = &tlsOpts{
  1100					httpsCert: httpsCert,
  1101					httpsKey:  httpsKey,
  1102				}
  1103			}
  1104			if err := b.addScanCabConfig(tlsO); err != nil {
  1105				return nil, fmt.Errorf("Could not generate config for scanning cabinet: %v", err)
  1106			}
  1107		}
  1108	
  1109		if b.runIndex() {
  1110			b.addUIConfig()
  1111			sto, err := b.sortedStorage("index")
  1112			if err != nil {
  1113				return nil, err
  1114			}
  1115			b.addPrefix("/index/", "storage-index", args{
  1116				"blobSource": "/bs/",
  1117				"storage":    sto,
  1118			})
  1119		}
  1120	
  1121		if conf.S3 != "" {
  1122			if err := b.addS3Config(conf.S3, "s3"); err != nil {
  1123				return nil, err
  1124			}
  1125		}
  1126		if conf.B2 != "" {
  1127			if err := b.addB2Config(conf.B2); err != nil {
  1128				return nil, err
  1129			}
  1130		}
  1131		if conf.GoogleDrive != "" {
  1132			if err := b.addGoogleDriveConfig(conf.GoogleDrive); err != nil {
  1133				return nil, err
  1134			}
  1135		}
  1136		if conf.GoogleCloudStorage != "" {
  1137			if err := b.addGoogleCloudStorageConfig(conf.GoogleCloudStorage); err != nil {
  1138				return nil, err
  1139			}
  1140		}
  1141	
  1142		return &Config{jconf: b.low}, nil
  1143	}
  1144	
  1145	func numSet(vv ...interface{}) (num int) {
  1146		for _, vi := range vv {
  1147			switch v := vi.(type) {
  1148			case string:
  1149				if v != "" {
  1150					num++
  1151				}
  1152			case bool:
  1153				if v {
  1154					num++
  1155				}
  1156			default:
  1157				panic("unknown type")
  1158			}
  1159		}
  1160		return
  1161	}
  1162	
  1163	var defaultBaseConfig = serverconfig.Config{
  1164		Listen: ":3179",
  1165		HTTPS:  false,
  1166		Auth:   "localhost",
  1167	}
  1168	
  1169	// WriteDefaultConfigFile generates a new default high-level server configuration
  1170	// file at filePath. The default indexer will use SQLite.
  1171	// If filePath already exists, it is overwritten.
  1172	func WriteDefaultConfigFile(filePath string) error {
  1173		conf := defaultBaseConfig
  1174		blobDir, err := osutil.CamliBlobRoot()
  1175		if err != nil {
  1176			return err
  1177		}
  1178		varDir, err := osutil.CamliVarDir()
  1179		if err != nil {
  1180			return err
  1181		}
  1182		if err := wkfs.MkdirAll(blobDir, 0700); err != nil {
  1183			return fmt.Errorf("Could not create default blobs directory: %v", err)
  1184		}
  1185		conf.BlobPath = blobDir
  1186		conf.PackRelated = true
  1187	
  1188		conf.SQLite = filepath.Join(varDir, "index.sqlite")
  1189	
  1190		keyID, secretRing, err := getOrMakeKeyring()
  1191		if err != nil {
  1192			return err
  1193		}
  1194		conf.Identity = keyID
  1195		conf.IdentitySecretRing = secretRing
  1196	
  1197		confData, err := json.MarshalIndent(conf, "", "    ")
  1198		if err != nil {
  1199			return fmt.Errorf("Could not json encode config file : %v", err)
  1200		}
  1201	
  1202		if err := wkfs.WriteFile(filePath, confData, 0600); err != nil {
  1203			return fmt.Errorf("Could not create or write default server config: %v", err)
  1204		}
  1205	
  1206		return nil
  1207	}
  1208	
  1209	func getOrMakeKeyring() (keyID, secRing string, err error) {
  1210		secRing = osutil.SecretRingFile()
  1211		_, err = wkfs.Stat(secRing)
  1212		switch {
  1213		case err == nil:
  1214			keyID, err = jsonsign.KeyIdFromRing(secRing)
  1215			if err != nil {
  1216				err = fmt.Errorf("Could not find any keyID in file %q: %v", secRing, err)
  1217				return
  1218			}
  1219			log.Printf("Re-using identity with keyID %q found in file %s", keyID, secRing)
  1220		case os.IsNotExist(err):
  1221			keyID, err = jsonsign.GenerateNewSecRing(secRing)
  1222			if err != nil {
  1223				err = fmt.Errorf("Could not generate new secRing at file %q: %v", secRing, err)
  1224				return
  1225			}
  1226			log.Printf("Generated new identity with keyID %q in file %s", keyID, secRing)
  1227		default:
  1228			err = fmt.Errorf("Could not stat secret ring %q: %v", secRing, err)
  1229		}
  1230		return
  1231	}