1	/*
     2	Copyright 2011 The Perkeep Authors.
     3	
     4	Licensed under the Apache License, Version 2.0 (the "License");
     5	you may not use this file except in compliance with the License.
     6	You may obtain a copy of the License at
     7	
     8	     http://www.apache.org/licenses/LICENSE-2.0
     9	
    10	Unless required by applicable law or agreed to in writing, software
    11	distributed under the License is distributed on an "AS IS" BASIS,
    12	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13	See the License for the specific language governing permissions and
    14	limitations under the License.
    15	*/
    16	
    17	package client
    18	
    19	import (
    20		"errors"
    21		"flag"
    22		"fmt"
    23		"log"
    24		"os"
    25		"path/filepath"
    26		"strconv"
    27		"strings"
    28		"sync"
    29	
    30		"go4.org/jsonconfig"
    31		"perkeep.org/internal/osutil"
    32		"perkeep.org/pkg/auth"
    33		"perkeep.org/pkg/blob"
    34		"perkeep.org/pkg/buildinfo"
    35		"perkeep.org/pkg/client/android"
    36		"perkeep.org/pkg/env"
    37		"perkeep.org/pkg/jsonsign"
    38		"perkeep.org/pkg/types/camtypes"
    39		"perkeep.org/pkg/types/clientconfig"
    40	
    41		"go4.org/wkfs"
    42	)
    43	
    44	// If set, flagServer overrides the JSON config file
    45	// ~/.config/perkeep/client-config.json
    46	// (i.e. osutil.UserClientConfigPath()) "server" key.
    47	//
    48	// A main binary must call AddFlags to expose it.
    49	var flagServer string
    50	
    51	// AddFlags registers the "server" and "secret-keyring" string flags.
    52	func AddFlags() {
    53		defaultPath := "/x/y/z/we're/in-a-test"
    54		if !buildinfo.TestingLinked() {
    55			defaultPath = osutil.UserClientConfigPath()
    56		}
    57		flag.StringVar(&flagServer, "server", "", "Perkeep server prefix. If blank, the default from the \"server\" field of "+defaultPath+" is used. Acceptable forms: https://you.example.com, example.com:1345 (https assumed), or http://you.example.com/alt-root")
    58		osutil.AddSecretRingFlag()
    59	}
    60	
    61	// ExplicitServer returns the Perkeep server given in the "server"
    62	// flag, if any.
    63	//
    64	// Use AddFlags to register the flag before any flag.Parse call.
    65	func ExplicitServer() string {
    66		return flagServer
    67	}
    68	
    69	var (
    70		configOnce sync.Once
    71		config     *clientconfig.Config
    72	
    73		configDisabled, _ = strconv.ParseBool(os.Getenv("CAMLI_DISABLE_CLIENT_CONFIG_FILE"))
    74	)
    75	
    76	// config parsing in the global environment.
    77	func parseConfig() {
    78		var nilClient *Client
    79		nilClient.parseConfig()
    80	}
    81	
    82	// lazy config parsing when there's a known client already.
    83	// The client c may be nil.
    84	func (c *Client) parseConfig() {
    85		if android.OnAndroid() {
    86			panic("parseConfig should never have been called on Android")
    87		}
    88		if configDisabled {
    89			panic("parseConfig should never have been called with CAMLI_DISABLE_CLIENT_CONFIG_FILE set")
    90		}
    91		configPath := osutil.UserClientConfigPath()
    92		if _, err := wkfs.Stat(configPath); os.IsNotExist(err) {
    93			if c != nil && c.isSharePrefix {
    94				return
    95			}
    96			errMsg := fmt.Sprintf("Client configuration file %v does not exist. See 'pk-put init' to generate it.", configPath)
    97			if keyID := serverKeyId(); keyID != "" {
    98				hint := fmt.Sprintf("\nThe key id %v was found in the server config %v, so you might want:\n'pk-put init -gpgkey %v'", keyID, osutil.UserServerConfigPath(), keyID)
    99				errMsg += hint
   100			}
   101			log.Fatal(errMsg)
   102		}
   103		// TODO: instead of using jsonconfig, we could read the file,
   104		// and unmarshal into the structs that we now have in
   105		// pkg/types/clientconfig. But we'll have to add the old
   106		// fields (before the name changes, and before the
   107		// multi-servers change) to the structs as well for our
   108		// graceful conversion/error messages to work.
   109		conf, err := osutil.NewJSONConfigParser().ReadFile(configPath)
   110		if err != nil {
   111			log.Fatal(err.Error())
   112		}
   113		cfg := jsonconfig.Obj(conf)
   114	
   115		if singleServerAuth := cfg.OptionalString("auth", ""); singleServerAuth != "" {
   116			newConf, err := convertToMultiServers(cfg)
   117			if err != nil {
   118				log.Print(err)
   119			} else {
   120				cfg = newConf
   121			}
   122		}
   123	
   124		config = &clientconfig.Config{
   125			Identity:           cfg.OptionalString("identity", ""),
   126			IdentitySecretRing: cfg.OptionalString("identitySecretRing", ""),
   127			IgnoredFiles:       cfg.OptionalList("ignoredFiles"),
   128		}
   129		serversList := make(map[string]*clientconfig.Server)
   130		servers := cfg.OptionalObject("servers")
   131		for alias, vei := range servers {
   132			// An alias should never be confused with a host name,
   133			// so we forbid anything looking like one.
   134			if isURLOrHostPort(alias) {
   135				log.Fatalf("Server alias %q looks like a hostname; \".\" or \";\" are not allowed.", alias)
   136			}
   137			serverMap, ok := vei.(map[string]interface{})
   138			if !ok {
   139				log.Fatalf("entry %q in servers section is a %T, want an object", alias, vei)
   140			}
   141			serverConf := jsonconfig.Obj(serverMap)
   142			serverStr, err := cleanServer(serverConf.OptionalString("server", ""))
   143			if err != nil {
   144				log.Fatalf("invalid server alias %q: %v", alias, err)
   145			}
   146			server := &clientconfig.Server{
   147				Server:       serverStr,
   148				Auth:         serverConf.OptionalString("auth", ""),
   149				IsDefault:    serverConf.OptionalBool("default", false),
   150				TrustedCerts: serverConf.OptionalList("trustedCerts"),
   151			}
   152			if err := serverConf.Validate(); err != nil {
   153				log.Fatalf("Error in servers section of config file for server %q: %v", alias, err)
   154			}
   155			serversList[alias] = server
   156		}
   157		config.Servers = serversList
   158		if err := cfg.Validate(); err != nil {
   159			printConfigChangeHelp(cfg)
   160			log.Fatalf("Error in config file: %v", err)
   161		}
   162	}
   163	
   164	// isURLOrHostPort returns true if s looks like a URL, or a hostname, i.e it starts with a scheme and/or it contains a period or a colon.
   165	func isURLOrHostPort(s string) bool {
   166		return strings.HasPrefix(s, "http://") ||
   167			strings.HasPrefix(s, "https://") ||
   168			strings.Contains(s, ".") || strings.Contains(s, ":")
   169	}
   170	
   171	// convertToMultiServers takes an old style single-server client configuration and maps it to new a multi-servers configuration that is returned.
   172	func convertToMultiServers(conf jsonconfig.Obj) (jsonconfig.Obj, error) {
   173		server := conf.OptionalString("server", "")
   174		if server == "" {
   175			return nil, errors.New("could not convert config to multi-servers style: no \"server\" key found")
   176		}
   177		newConf := jsonconfig.Obj{
   178			"servers": map[string]interface{}{
   179				"server": map[string]interface{}{
   180					"auth":    conf.OptionalString("auth", ""),
   181					"default": true,
   182					"server":  server,
   183				},
   184			},
   185			"identity":           conf.OptionalString("identity", ""),
   186			"identitySecretRing": conf.OptionalString("identitySecretRing", ""),
   187		}
   188		if ignoredFiles := conf.OptionalList("ignoredFiles"); ignoredFiles != nil {
   189			var list []interface{}
   190			for _, v := range ignoredFiles {
   191				list = append(list, v)
   192			}
   193			newConf["ignoredFiles"] = list
   194		}
   195		return newConf, nil
   196	}
   197	
   198	// printConfigChangeHelp checks if conf contains obsolete keys,
   199	// and prints additional help in this case.
   200	func printConfigChangeHelp(conf jsonconfig.Obj) {
   201		// rename maps from old key names to the new ones.
   202		// If there is no new one, the value is the empty string.
   203		rename := map[string]string{
   204			"keyId":            "identity",
   205			"publicKeyBlobref": "",
   206			"selfPubKeyDir":    "",
   207			"secretRing":       "identitySecretRing",
   208		}
   209		oldConfig := false
   210		configChangedMsg := fmt.Sprintf("The client configuration file (%s) keys have changed.\n", osutil.UserClientConfigPath())
   211		for _, unknown := range conf.UnknownKeys() {
   212			v, ok := rename[unknown]
   213			if ok {
   214				if v != "" {
   215					configChangedMsg += fmt.Sprintf("%q should be renamed %q.\n", unknown, v)
   216				} else {
   217					configChangedMsg += fmt.Sprintf("%q should be removed.\n", unknown)
   218				}
   219				oldConfig = true
   220			}
   221		}
   222		if oldConfig {
   223			configChangedMsg += "Please see https://perkeep.org/doc/client-config, or use pk-put init to recreate a default one."
   224			log.Print(configChangedMsg)
   225		}
   226	}
   227	
   228	// serverKeyId returns the public gpg key id ("identity" field)
   229	// from the user's server config , if any.
   230	// It returns the empty string otherwise.
   231	func serverKeyId() string {
   232		serverConfigFile := osutil.UserServerConfigPath()
   233		if _, err := wkfs.Stat(serverConfigFile); err != nil {
   234			if os.IsNotExist(err) {
   235				return ""
   236			}
   237			log.Fatalf("Could not stat %v: %v", serverConfigFile, err)
   238		}
   239		obj, err := jsonconfig.ReadFile(serverConfigFile)
   240		if err != nil {
   241			return ""
   242		}
   243		keyID, ok := obj["identity"].(string)
   244		if !ok {
   245			return ""
   246		}
   247		return keyID
   248	}
   249	
   250	// cleanServer returns the canonical URL of the provided server, which must be a URL, IP, host (with dot), or host/ip:port.
   251	// The returned canonical URL will have trailing slashes removed and be prepended with "https://" if no scheme is provided.
   252	func cleanServer(server string) (string, error) {
   253		if !isURLOrHostPort(server) {
   254			return "", fmt.Errorf("server %q does not look like a server address and could be confused with a server alias. It should look like [http[s]://]foo[.com][:port] with at least one of the optional parts.", server)
   255		}
   256		// Remove trailing slash if provided.
   257		if strings.HasSuffix(server, "/") {
   258			server = server[0 : len(server)-1]
   259		}
   260		// Default to "https://" when not specified
   261		if !strings.HasPrefix(server, "http") && !strings.HasPrefix(server, "https") {
   262			server = "https://" + server
   263		}
   264		return server, nil
   265	}
   266	
   267	// getServer returns the server's URL found either as a command-line flag,
   268	// or as the default server in the config file.
   269	func getServer() (string, error) {
   270		if s := os.Getenv("CAMLI_SERVER"); s != "" {
   271			return cleanServer(s)
   272		}
   273		if flagServer != "" {
   274			if !isURLOrHostPort(flagServer) {
   275				configOnce.Do(parseConfig)
   276				serverConf, ok := config.Servers[flagServer]
   277				if ok {
   278					return serverConf.Server, nil
   279				}
   280				log.Printf("%q looks like a server alias, but no such alias found in config.", flagServer)
   281			} else {
   282				return cleanServer(flagServer)
   283			}
   284		}
   285		server, err := defaultServer()
   286		if err != nil {
   287			return "", err
   288		}
   289		if server == "" {
   290			return "", camtypes.ErrClientNoServer
   291		}
   292		return cleanServer(server)
   293	}
   294	
   295	func defaultServer() (string, error) {
   296		configOnce.Do(parseConfig)
   297		wantAlias := os.Getenv("CAMLI_DEFAULT_SERVER")
   298		for alias, serverConf := range config.Servers {
   299			if (wantAlias != "" && wantAlias == alias) || (wantAlias == "" && serverConf.IsDefault) {
   300				return cleanServer(serverConf.Server)
   301			}
   302		}
   303		return "", nil
   304	}
   305	
   306	func (c *Client) useTLS() bool {
   307		return strings.HasPrefix(c.discoRoot(), "https://")
   308	}
   309	
   310	// SetupAuth sets the client's authMode. It tries from the environment first if we're on android or in dev mode, and then from the client configuration.
   311	func (c *Client) SetupAuth() error {
   312		if c.noExtConfig {
   313			if c.authMode != nil {
   314				if _, ok := c.authMode.(*auth.None); !ok {
   315					return nil
   316				}
   317			}
   318			return errors.New("client: noExtConfig set; auth should not be configured from config or env vars")
   319		}
   320		// env var takes precedence, but only if we're in dev mode or on android.
   321		// Too risky otherwise.
   322		if android.OnAndroid() ||
   323			env.IsDev() ||
   324			configDisabled {
   325			authMode, err := auth.FromEnv()
   326			if err == nil {
   327				c.authMode = authMode
   328				return nil
   329			}
   330			if err != auth.ErrNoAuth {
   331				return fmt.Errorf("Could not set up auth from env var CAMLI_AUTH: %v", err)
   332			}
   333		}
   334		if c.server == "" {
   335			return fmt.Errorf("no server defined for this client: can not set up auth")
   336		}
   337		authConf := serverAuth(c.server)
   338		if authConf == "" {
   339			c.authErr = fmt.Errorf("could not find auth key for server %q in config, defaulting to no auth", c.server)
   340			c.authMode = auth.None{}
   341			return nil
   342		}
   343		var err error
   344		c.authMode, err = auth.FromConfig(authConf)
   345		return err
   346	}
   347	
   348	// serverAuth returns the auth scheme for server from the config, or the empty string if the server was not found in the config.
   349	func serverAuth(server string) string {
   350		configOnce.Do(parseConfig)
   351		alias := config.Alias(server)
   352		if alias == "" {
   353			return ""
   354		}
   355		return config.Servers[alias].Auth
   356	}
   357	
   358	// SetupAuthFromString configures the clients authentication mode from
   359	// an explicit auth string.
   360	func (c *Client) SetupAuthFromString(a string) error {
   361		// TODO(mpl): review the one using that (pkg/blobserver/remote/remote.go)
   362		var err error
   363		c.authMode, err = auth.FromConfig(a)
   364		return err
   365	}
   366	
   367	// SecretRingFile returns the filename to the user's GPG secret ring.
   368	// The value comes from either the --secret-keyring flag, the
   369	// CAMLI_SECRET_RING environment variable, the client config file's
   370	// "identitySecretRing" value, or the operating system default location.
   371	func (c *Client) SecretRingFile() string {
   372		if osutil.HasSecretRingFlag() {
   373			if secretRing, ok := osutil.ExplicitSecretRingFile(); ok {
   374				return secretRing
   375			}
   376		}
   377		if android.OnAndroid() {
   378			panic("on android, so CAMLI_SECRET_RING should have been defined, or --secret-keyring used.")
   379		}
   380		if c.noExtConfig {
   381			log.Print("client: noExtConfig set; cannot get secret ring file from config or env vars.")
   382			return ""
   383		}
   384		if configDisabled {
   385			panic("Need a secret ring, and config file disabled")
   386		}
   387		configOnce.Do(parseConfig)
   388		if config.IdentitySecretRing == "" {
   389			return osutil.SecretRingFile()
   390		}
   391		return config.IdentitySecretRing
   392	}
   393	
   394	func fileExists(name string) bool {
   395		_, err := os.Stat(name)
   396		return err == nil
   397	}
   398	
   399	// SignerPublicKeyBlobref returns the blobref of signer's public key.
   400	// The blobref may not be valid (zero blob.Ref) if e.g the configuration
   401	// is invalid or incomplete.
   402	func (c *Client) SignerPublicKeyBlobref() blob.Ref {
   403		c.initSignerPublicKeyBlobrefOnce.Do(c.initSignerPublicKeyBlobref)
   404		return c.signerPublicKeyRef
   405	}
   406	
   407	func (c *Client) initSignerPublicKeyBlobref() {
   408		if c.noExtConfig {
   409			log.Print("client: noExtConfig set; cannot get public key from config or env vars.")
   410			return
   411		}
   412		keyID := os.Getenv("CAMLI_KEYID")
   413		if keyID == "" {
   414			configOnce.Do(parseConfig)
   415			keyID = config.Identity
   416			if keyID == "" {
   417				log.Fatalf("No 'identity' key in JSON configuration file %q; have you run \"pk-put init\"?", osutil.UserClientConfigPath())
   418			}
   419		}
   420		keyRing := c.SecretRingFile()
   421		if !fileExists(keyRing) {
   422			log.Fatalf("Could not find keyID %q, because secret ring file %q does not exist.", keyID, keyRing)
   423		}
   424		entity, err := jsonsign.EntityFromSecring(keyID, keyRing)
   425		if err != nil {
   426			log.Fatalf("Couldn't find keyID %q in secret ring %v: %v", keyID, keyRing, err)
   427		}
   428		armored, err := jsonsign.ArmoredPublicKey(entity)
   429		if err != nil {
   430			log.Fatalf("Error serializing public key: %v", err)
   431		}
   432	
   433		c.signerPublicKeyRef = blob.RefFromString(armored)
   434		c.publicKeyArmored = armored
   435	}
   436	
   437	func (c *Client) initTrustedCerts() {
   438		if c.noExtConfig {
   439			return
   440		}
   441		if e := os.Getenv("CAMLI_TRUSTED_CERT"); e != "" {
   442			c.trustedCerts = strings.Split(e, ",")
   443			return
   444		}
   445		c.trustedCerts = []string{}
   446		if android.OnAndroid() || configDisabled {
   447			return
   448		}
   449		if c.server == "" {
   450			log.Printf("No server defined: can not define trustedCerts for this client.")
   451			return
   452		}
   453		trustedCerts := c.serverTrustedCerts(c.server)
   454		if trustedCerts == nil {
   455			return
   456		}
   457		for _, trustedCert := range trustedCerts {
   458			c.trustedCerts = append(c.trustedCerts, strings.ToLower(trustedCert))
   459		}
   460	}
   461	
   462	// serverTrustedCerts returns the trusted certs for server from the config.
   463	func (c *Client) serverTrustedCerts(server string) []string {
   464		configOnce.Do(c.parseConfig)
   465		if config == nil {
   466			return nil
   467		}
   468		alias := config.Alias(server)
   469		if alias == "" {
   470			return nil
   471		}
   472		return config.Servers[alias].TrustedCerts
   473	}
   474	
   475	func (c *Client) getTrustedCerts() []string {
   476		c.initTrustedCertsOnce.Do(c.initTrustedCerts)
   477		return c.trustedCerts
   478	}
   479	
   480	func (c *Client) initIgnoredFiles() {
   481		defer func() {
   482			c.ignoreChecker = newIgnoreChecker(c.ignoredFiles)
   483		}()
   484		if c.noExtConfig {
   485			return
   486		}
   487		if e := os.Getenv("CAMLI_IGNORED_FILES"); e != "" {
   488			c.ignoredFiles = strings.Split(e, ",")
   489			return
   490		}
   491		c.ignoredFiles = []string{}
   492		if android.OnAndroid() || configDisabled {
   493			return
   494		}
   495		configOnce.Do(parseConfig)
   496		c.ignoredFiles = config.IgnoredFiles
   497	}
   498	
   499	var osutilHomeDir = osutil.HomeDir // changed by tests
   500	
   501	// newIgnoreChecker uses ignoredFiles to build and return a func that returns whether the file path argument should be ignored. See IsIgnoredFile for the ignore rules.
   502	func newIgnoreChecker(ignoredFiles []string) func(path string) (shouldIgnore bool) {
   503		var fns []func(string) bool
   504	
   505		// copy of ignoredFiles for us to mutate
   506		ignFiles := append([]string(nil), ignoredFiles...)
   507		for k, v := range ignFiles {
   508			if strings.HasPrefix(v, filepath.FromSlash("~/")) {
   509				ignFiles[k] = filepath.Join(osutilHomeDir(), v[2:])
   510			}
   511		}
   512		// We cache the ignoredFiles patterns in 3 categories (not necessarily exclusive):
   513		// 1) shell patterns
   514		// 3) absolute paths
   515		// 4) paths components
   516		for _, pattern := range ignFiles {
   517			pattern := pattern
   518			_, err := filepath.Match(pattern, "whatever")
   519			if err == nil {
   520				fns = append(fns, func(v string) bool { return isShellPatternMatch(pattern, v) })
   521			}
   522		}
   523		for _, pattern := range ignFiles {
   524			pattern := pattern
   525			if filepath.IsAbs(pattern) {
   526				fns = append(fns, func(v string) bool { return hasDirPrefix(filepath.Clean(pattern), v) })
   527			} else {
   528				fns = append(fns, func(v string) bool { return hasComponent(filepath.Clean(pattern), v) })
   529			}
   530		}
   531	
   532		return func(path string) bool {
   533			for _, fn := range fns {
   534				if fn(path) {
   535					return true
   536				}
   537			}
   538			return false
   539		}
   540	}
   541	
   542	var filepathSeparatorString = string(filepath.Separator)
   543	
   544	// isShellPatternMatch returns whether fullpath matches the shell pattern, as defined by http://golang.org/pkg/path/filepath/#Match. As an additional special case, when the pattern looks like a basename, the last path element of fullpath is also checked against it.
   545	func isShellPatternMatch(shellPattern, fullpath string) bool {
   546		match, _ := filepath.Match(shellPattern, fullpath)
   547		if match {
   548			return true
   549		}
   550		if !strings.Contains(shellPattern, filepathSeparatorString) {
   551			match, _ := filepath.Match(shellPattern, filepath.Base(fullpath))
   552			if match {
   553				return true
   554			}
   555		}
   556		return false
   557	}
   558	
   559	// hasDirPrefix reports whether the path has the provided directory prefix.
   560	// Both should be absolute paths.
   561	func hasDirPrefix(dirPrefix, fullpath string) bool {
   562		if !strings.HasPrefix(fullpath, dirPrefix) {
   563			return false
   564		}
   565		if len(fullpath) == len(dirPrefix) {
   566			return true
   567		}
   568		if fullpath[len(dirPrefix)] == filepath.Separator {
   569			return true
   570		}
   571		return false
   572	}
   573	
   574	// hasComponent returns whether the pathComponent is a path component of
   575	// fullpath. i.e it is a part of fullpath that fits exactly between two path
   576	// separators.
   577	func hasComponent(component, fullpath string) bool {
   578		// trim Windows volume name
   579		fullpath = strings.TrimPrefix(fullpath, filepath.VolumeName(fullpath))
   580		for {
   581			i := strings.Index(fullpath, component)
   582			if i == -1 {
   583				return false
   584			}
   585			if i != 0 && fullpath[i-1] == filepath.Separator {
   586				componentEnd := i + len(component)
   587				if componentEnd == len(fullpath) {
   588					return true
   589				}
   590				if fullpath[componentEnd] == filepath.Separator {
   591					return true
   592				}
   593			}
   594			fullpath = fullpath[i+1:]
   595		}
   596	}