1	/*
     2	Copyright 2011 The Perkeep Authors.
     3	
     4	Licensed under the Apache License, Version 2.0 (the "License");
     5	you may not use this file except in compliance with the License.
     6	You may obtain a copy of the License at
     7	
     8	     http://www.apache.org/licenses/LICENSE-2.0
     9	
    10	Unless required by applicable law or agreed to in writing, software
    11	distributed under the License is distributed on an "AS IS" BASIS,
    12	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13	See the License for the specific language governing permissions and
    14	limitations under the License.
    15	*/
    16	
    17	// Package client implements a Perkeep client.
    18	package client // import "perkeep.org/pkg/client"
    19	
    20	import (
    21		"bytes"
    22		"context"
    23		"crypto/tls"
    24		"encoding/json"
    25		"errors"
    26		"fmt"
    27		"io"
    28		"log"
    29		"net"
    30		"net/http"
    31		"net/url"
    32		"os"
    33		"regexp"
    34		"strings"
    35		"sync"
    36		"time"
    37	
    38		"perkeep.org/internal/hashutil"
    39		"perkeep.org/internal/httputil"
    40		"perkeep.org/internal/osutil"
    41		"perkeep.org/pkg/auth"
    42		"perkeep.org/pkg/blob"
    43		"perkeep.org/pkg/blobserver"
    44		"perkeep.org/pkg/buildinfo"
    45		"perkeep.org/pkg/client/android"
    46		"perkeep.org/pkg/schema"
    47		"perkeep.org/pkg/search"
    48		"perkeep.org/pkg/types/camtypes"
    49	
    50		"go4.org/syncutil"
    51		"golang.org/x/net/http2"
    52	)
    53	
    54	// A Client provides access to a Perkeep server.
    55	//
    56	// After use, a Client should be closed via its Close method to
    57	// release idle HTTP connections or other resourcedds.
    58	type Client struct {
    59		// server is the input from user, pre-discovery.
    60		// For example "http://foo.com" or "foo.com:1234".
    61		// It is the responsibility of initPrefix to parse
    62		// server and set prefix, including doing discovery
    63		// to figure out what the proper server-declared
    64		// prefix is.
    65		server string
    66	
    67		prefixOnce    syncutil.Once // guards init of following 2 fields
    68		prefixv       string        // URL prefix before "/camli/"
    69		isSharePrefix bool          // URL is a request for a share blob
    70	
    71		discoOnce              syncutil.Once
    72		searchRoot             string      // Handler prefix, or "" if none
    73		downloadHelper         string      // or "" if none
    74		storageGen             string      // storage generation, or "" if not reported
    75		hasLegacySHA1          bool        // Whether server has SHA-1 blobs indexed.
    76		syncHandlers           []*SyncInfo // "from" and "to" url prefix for each syncHandler
    77		serverKeyID            string      // Server's GPG public key ID.
    78		helpRoot               string      // Handler prefix, or "" if none
    79		shareRoot              string      // Share handler prefix, or "" if none
    80		serverPublicKeyBlobRef blob.Ref    // Server's public key blobRef
    81	
    82		signerOnce  sync.Once
    83		signer      *schema.Signer
    84		signerErr   error
    85		signHandler string // Handler prefix, or "" if none
    86	
    87		authMode auth.AuthMode
    88		// authErr is set when no auth config is found but we want to defer warning
    89		// until discovery fails.
    90		authErr error
    91	
    92		httpClient *http.Client
    93		haveCache  HaveCache
    94	
    95		// If sto is set, it's used before the httpClient or other network operations.
    96		sto blobserver.Storage
    97	
    98		initTrustedCertsOnce sync.Once
    99	
   100		// We define a certificate fingerprint as the 20 digits lowercase prefix
   101		// of the SHA256 of the complete certificate (in ASN.1 DER encoding).
   102		// trustedCerts contains the fingerprints of the self-signed
   103		// certificates we trust.
   104		// If not empty, (and if using TLS) the full x509 verification is
   105		// disabled, and we instead check the server's certificate against
   106		// this list.
   107		// The perkeepd server prints the fingerprint to add to the config
   108		// when starting.
   109		trustedCerts []string
   110	
   111		// insecureAnyTLSCert disables all TLS cert checking,
   112		// including the trustedCerts field above.
   113		insecureAnyTLSCert bool
   114	
   115		initIgnoredFilesOnce sync.Once
   116		// list of files that pk-put should ignore.
   117		// Defaults to empty, but pk-put init creates a config with a non
   118		// empty list.
   119		// See IsIgnoredFile for the matching rules.
   120		ignoredFiles  []string
   121		ignoreChecker func(path string) bool
   122	
   123		initSignerPublicKeyBlobrefOnce sync.Once
   124		signerPublicKeyRef             blob.Ref
   125		publicKeyArmored               string
   126	
   127		statsMutex sync.Mutex
   128		stats      Stats
   129	
   130		// via maps the access path from a share root to a desired target.
   131		// It is non-nil when in "sharing" mode, where the Client is fetching
   132		// a share.
   133		viaMu sync.RWMutex
   134		via   map[blob.Ref]blob.Ref // target => via (target is referenced from via)
   135	
   136		// Verbose controls how much logging from the client is printed. The caller
   137		// should set it only before using the client, and treat it as read-only after
   138		// that.
   139		Verbose bool
   140		// Logger is the logger used by the client. It defaults to a standard
   141		// logger to os.Stderr if the client is initialized by one of the package's
   142		// functions. Like Verbose, it should be set only before using the client, and
   143		// not be modified afterwards.
   144		Logger *log.Logger
   145	
   146		httpGate        *syncutil.Gate
   147		transportConfig *TransportConfig // or nil
   148	
   149		noExtConfig bool // no external config; config file and env vars are ignored.
   150	
   151		// sameOrigin indicates whether URLs in requests should be stripped from
   152		// their scheme and HostPort parts. This is meant for when using the client
   153		// through gopherjs in the web UI. Because we'll run into CORS errors if
   154		// requests have a Host part.
   155		sameOrigin bool
   156	}
   157	
   158	const maxParallelHTTP_h1 = 5
   159	const maxParallelHTTP_h2 = 50
   160	
   161	// New returns a new Perkeep Client.
   162	//
   163	// By default, with no options, it uses the client as configured in
   164	// the environment or default configuration files.
   165	func New(opts ...ClientOption) (*Client, error) {
   166		c := &Client{
   167			haveCache: noHaveCache{},
   168			Logger:    log.New(os.Stderr, "", log.Ldate|log.Ltime),
   169			authMode:  auth.None{},
   170		}
   171		for _, v := range opts {
   172			v.modifyClient(c)
   173		}
   174		if c.sto != nil && len(opts) > 1 {
   175			return nil, errors.New("use of OptionUseStorageClient precludes use of any other options")
   176		}
   177	
   178		if c.noExtConfig {
   179			c.setDefaultHTTPClient()
   180			return c, nil
   181		}
   182	
   183		if c.server != "" {
   184			if !isURLOrHostPort(c.server) {
   185				configOnce.Do(parseConfig)
   186				serverConf, ok := config.Servers[c.server]
   187				if !ok {
   188					log.Fatalf("%q looks like a server alias, but no such alias found in config at %v", c.server, osutil.UserClientConfigPath())
   189				}
   190				c.server = serverConf.Server
   191			}
   192			c.setDefaultHTTPClient()
   193			return c, nil
   194		}
   195	
   196		var err error
   197		c.server, err = getServer()
   198		if err != nil {
   199			return nil, err
   200		}
   201		err = c.SetupAuth()
   202		if err != nil {
   203			return nil, err
   204		}
   205		c.setDefaultHTTPClient()
   206		return c, nil
   207	}
   208	
   209	func (c *Client) setDefaultHTTPClient() {
   210		if c.httpClient == nil {
   211			c.httpClient = &http.Client{
   212				Transport: c.transportForConfig(c.transportConfig),
   213			}
   214		}
   215		c.httpGate = syncutil.NewGate(httpGateSize(c.httpClient.Transport))
   216	}
   217	
   218	// NewOrFail is like New, but calls log.Fatal instead of returning an error.
   219	func NewOrFail(opts ...ClientOption) *Client {
   220		c, err := New(opts...)
   221		if err != nil {
   222			log.Fatalf("error creating client: %v", err)
   223		}
   224		return c
   225	}
   226	
   227	// NewPathClient returns a new client accessing a subpath of c.
   228	func (c *Client) NewPathClient(path string) (*Client, error) {
   229		u, err := url.Parse(c.server)
   230		if err != nil {
   231			return nil, fmt.Errorf("bogus server %q for NewPathClient receiver: %v", c.server, err)
   232		}
   233		u.Path = path
   234		pc, err := New(OptionServer(u.String()))
   235		if err != nil {
   236			return nil, err
   237		}
   238		pc.authMode = c.authMode
   239		pc.discoOnce.Do(noop)
   240		return pc, nil
   241	}
   242	
   243	// TransportConfig contains options for how HTTP requests are made.
   244	type TransportConfig struct {
   245		// Proxy optionally specifies the Proxy for the transport. Useful with
   246		// pk-put for debugging even localhost requests.
   247		Proxy   func(*http.Request) (*url.URL, error)
   248		Verbose bool // Verbose enables verbose logging of HTTP requests.
   249	}
   250	
   251	func (c *Client) useHTTP2(tc *TransportConfig) bool {
   252		if !c.useTLS() {
   253			return false
   254		}
   255		if android.IsChild() {
   256			// No particular reason; just untested so far.
   257			return false
   258		}
   259		if os.Getenv("HTTPS_PROXY") != "" || os.Getenv("https_proxy") != "" ||
   260			(tc != nil && tc.Proxy != nil) {
   261			// Also just untested. Which proxies support h2 anyway?
   262			return false
   263		}
   264		return true
   265	}
   266	
   267	// transportForConfig returns a transport for the client, setting the correct
   268	// Proxy, Dial, and TLSClientConfig if needed. It does not mutate c.
   269	// It is the caller's responsibility to then use that transport to set
   270	// the client's httpClient with SetHTTPClient.
   271	func (c *Client) transportForConfig(tc *TransportConfig) http.RoundTripper {
   272		if c == nil {
   273			return nil
   274		}
   275		var transport http.RoundTripper
   276		proxy := http.ProxyFromEnvironment
   277		if tc != nil && tc.Proxy != nil {
   278			proxy = tc.Proxy
   279		}
   280	
   281		if c.useHTTP2(tc) {
   282			transport = &http2.Transport{
   283				DialTLS: c.http2DialTLSFunc(),
   284			}
   285		} else {
   286			transport = &http.Transport{
   287				DialTLS:             c.DialTLSFunc(),
   288				Dial:                c.DialFunc(),
   289				Proxy:               proxy,
   290				MaxIdleConnsPerHost: maxParallelHTTP_h1,
   291			}
   292		}
   293		httpStats := &httputil.StatsTransport{
   294			Transport: transport,
   295		}
   296		if tc != nil {
   297			httpStats.VerboseLog = tc.Verbose
   298		}
   299		transport = httpStats
   300		if android.IsChild() {
   301			transport = &android.StatsTransport{Rt: transport}
   302		}
   303		return transport
   304	}
   305	
   306	// HTTPStats returns the client's underlying httputil.StatsTransport, if in use.
   307	// If another transport is being used, nil is returned.
   308	func (c *Client) HTTPStats() *httputil.StatsTransport {
   309		st, _ := c.httpClient.Transport.(*httputil.StatsTransport)
   310		return st
   311	}
   312	
   313	type ClientOption interface {
   314		modifyClient(*Client)
   315	}
   316	
   317	// OptionServer returns a Client constructor option that forces use of
   318	// the provided server.
   319	//
   320	// The provided server is either "host:port" (assumed http, not https)
   321	// or a URL prefix, with or without a path, or a server alias from the
   322	// client configuration file. A server alias should not be confused
   323	// with a hostname, therefore it cannot contain any colon or period.
   324	func OptionServer(server string) ClientOption {
   325		return optionServer(server)
   326	}
   327	
   328	type optionServer string
   329	
   330	func (s optionServer) modifyClient(c *Client) { c.server = string(s) }
   331	
   332	// OptionUseStorageClient returns a Client constructor option that
   333	// forces use of the provided blob storage target.
   334	//
   335	// This exists mainly so all the convenience methods on
   336	// Client (e.g. the Upload variants) are available against storage
   337	// directly.
   338	//
   339	// Use of OptionUseStorageClient is mutually exclusively
   340	// with all other options, although it does imply
   341	// OptionNoExternalConfig(true).
   342	func OptionUseStorageClient(s blobserver.Storage) ClientOption {
   343		return optionStorage{s}
   344	}
   345	
   346	type optionStorage struct {
   347		sto blobserver.Storage
   348	}
   349	
   350	func (o optionStorage) modifyClient(c *Client) {
   351		c.sto = o.sto
   352		c.noExtConfig = true
   353	}
   354	
   355	// OptionTransportConfig returns a ClientOption that makes the client use
   356	// the provided transport configuration options.
   357	func OptionTransportConfig(tc *TransportConfig) ClientOption {
   358		return optionTransportConfig{tc}
   359	}
   360	
   361	type optionTransportConfig struct {
   362		tc *TransportConfig
   363	}
   364	
   365	func (o optionTransportConfig) modifyClient(c *Client) { c.transportConfig = o.tc }
   366	
   367	// OptionInsecure returns a ClientOption that controls whether HTTP
   368	// requests are allowed to be insecure (over HTTP or HTTPS without TLS
   369	// certificate checking). Use of this is strongly discouraged except
   370	// for local testing.
   371	func OptionInsecure(v bool) ClientOption {
   372		return optionInsecure(v)
   373	}
   374	
   375	type optionInsecure bool
   376	
   377	func (o optionInsecure) modifyClient(c *Client) { c.insecureAnyTLSCert = bool(o) }
   378	
   379	// OptionTrustedCert returns a ClientOption that makes the client
   380	// trust the provide self-signed cert signature. The value should be
   381	// the 20 byte hex prefix of the SHA-256 of the cert, as printed by
   382	// the perkeepd server on start-up.
   383	//
   384	// If cert is empty, the option has no effect.
   385	func OptionTrustedCert(cert string) ClientOption {
   386		// TODO: remove this whole function now that we have LetsEncrypt?
   387		return optionTrustedCert(cert)
   388	}
   389	
   390	type optionTrustedCert string
   391	
   392	func (o optionTrustedCert) modifyClient(c *Client) {
   393		cert := string(o)
   394		if cert != "" {
   395			c.initTrustedCertsOnce.Do(func() {})
   396			c.trustedCerts = []string{string(o)}
   397		}
   398	}
   399	
   400	type optionNoExtConfig bool
   401	
   402	func (o optionNoExtConfig) modifyClient(c *Client) { c.noExtConfig = bool(o) }
   403	
   404	// OptionNoExternalConfig returns a Client constructor option that
   405	// prevents any on-disk config files or environment variables from
   406	// influencing the client. It may still use the disk for caches.
   407	func OptionNoExternalConfig() ClientOption {
   408		return optionNoExtConfig(true)
   409	}
   410	
   411	type optionAuthMode struct {
   412		m auth.AuthMode
   413	}
   414	
   415	func (o optionAuthMode) modifyClient(c *Client) { c.authMode = o.m }
   416	
   417	// OptionAuthMode returns a Client constructor option that sets the
   418	// client's authentication mode.
   419	func OptionAuthMode(m auth.AuthMode) ClientOption {
   420		return optionAuthMode{m}
   421	}
   422	
   423	// noop is for use with syncutil.Onces.
   424	func noop() error { return nil }
   425	
   426	var shareURLRx = regexp.MustCompile(`^(.+)/(` + blob.Pattern + ")$")
   427	
   428	// NewFromShareRoot uses shareBlobURL to set up and return a client that
   429	// will be used to fetch shared blobs.
   430	func NewFromShareRoot(ctx context.Context, shareBlobURL string, opts ...ClientOption) (c *Client, target blob.Ref, err error) {
   431		var root string
   432		m := shareURLRx.FindStringSubmatch(shareBlobURL)
   433		if m == nil {
   434			return nil, blob.Ref{}, fmt.Errorf("Unknown share URL base")
   435		}
   436		c, err = New(append(opts[:len(opts):cap(opts)], OptionServer(m[1]))...)
   437		if err != nil {
   438			return nil, blob.Ref{}, err
   439		}
   440		c.discoOnce.Do(noop)
   441		c.prefixOnce.Do(noop)
   442		c.prefixv = m[1]
   443		c.isSharePrefix = true
   444		c.authMode = auth.None{}
   445		c.via = make(map[blob.Ref]blob.Ref)
   446		root = m[2]
   447	
   448		req := c.newRequest(ctx, "GET", shareBlobURL, nil)
   449		res, err := c.expect2XX(req)
   450		if err != nil {
   451			return nil, blob.Ref{}, fmt.Errorf("error fetching %s: %v", shareBlobURL, err)
   452		}
   453		defer res.Body.Close()
   454		var buf bytes.Buffer
   455		rootbr, ok := blob.Parse(root)
   456		if !ok {
   457			return nil, blob.Ref{}, fmt.Errorf("invalid root blob ref for sharing: %q", root)
   458		}
   459		b, err := schema.BlobFromReader(rootbr, io.TeeReader(res.Body, &buf))
   460		if err != nil {
   461			return nil, blob.Ref{}, fmt.Errorf("error parsing JSON from %s: %v , with response: %q", shareBlobURL, err, buf.Bytes())
   462		}
   463		if b.ShareAuthType() != schema.ShareHaveRef {
   464			return nil, blob.Ref{}, fmt.Errorf("unknown share authType of %q", b.ShareAuthType())
   465		}
   466		target = b.ShareTarget()
   467		if !target.Valid() {
   468			return nil, blob.Ref{}, fmt.Errorf("no target")
   469		}
   470		c.via[target] = rootbr
   471		return c, target, nil
   472	}
   473	
   474	// SetHTTPClient sets the Perkeep client's HTTP client.
   475	// If nil, the default HTTP client is used.
   476	func (c *Client) SetHTTPClient(client *http.Client) {
   477		if client == nil {
   478			client = http.DefaultClient
   479		}
   480		c.httpClient = client
   481	}
   482	
   483	// HTTPClient returns the Client's underlying http.Client.
   484	func (c *Client) HTTPClient() *http.Client {
   485		return c.httpClient
   486	}
   487	
   488	// A HaveCache caches whether a remote blobserver has a blob.
   489	type HaveCache interface {
   490		StatBlobCache(br blob.Ref) (size uint32, ok bool)
   491		NoteBlobExists(br blob.Ref, size uint32)
   492	}
   493	
   494	type noHaveCache struct{}
   495	
   496	func (noHaveCache) StatBlobCache(blob.Ref) (uint32, bool) { return 0, false }
   497	func (noHaveCache) NoteBlobExists(blob.Ref, uint32)       {}
   498	
   499	func (c *Client) SetHaveCache(cache HaveCache) {
   500		if cache == nil {
   501			cache = noHaveCache{}
   502		}
   503		c.haveCache = cache
   504	}
   505	
   506	func (c *Client) printf(format string, v ...interface{}) {
   507		if c.Verbose && c.Logger != nil {
   508			c.Logger.Printf(format, v...)
   509		}
   510	}
   511	
   512	func (c *Client) Stats() Stats {
   513		c.statsMutex.Lock()
   514		defer c.statsMutex.Unlock()
   515		return c.stats // copy
   516	}
   517	
   518	// ErrNoSearchRoot is returned by SearchRoot if the server doesn't support search.
   519	var ErrNoSearchRoot = errors.New("client: server doesn't support search")
   520	
   521	// ErrNoHelpRoot is returned by HelpRoot if the server doesn't have a help handler.
   522	var ErrNoHelpRoot = errors.New("client: server does not have a help handler")
   523	
   524	// ErrNoShareRoot is returned by ShareRoot if the server doesn't have a share handler.
   525	var ErrNoShareRoot = errors.New("client: server does not have a share handler")
   526	
   527	// ErrNoSigning is returned by ServerKeyID if the server doesn't support signing.
   528	var ErrNoSigning = fmt.Errorf("client: server doesn't support signing")
   529	
   530	// ErrNoStorageGeneration is returned by StorageGeneration if the
   531	// server doesn't report a storage generation value.
   532	var ErrNoStorageGeneration = errors.New("client: server doesn't report a storage generation")
   533	
   534	// ErrNoSync is returned by SyncHandlers if the server does not advertise syncs.
   535	var ErrNoSync = errors.New("client: server has no sync handlers")
   536	
   537	// BlobRoot returns the server's blobroot URL prefix.
   538	// If the client was constructed with an explicit path,
   539	// that path is used. Otherwise the server's
   540	// default advertised blobRoot is used.
   541	func (c *Client) BlobRoot() (string, error) {
   542		prefix, err := c.prefix()
   543		if err != nil {
   544			return "", err
   545		}
   546		return prefix + "/", nil
   547	}
   548	
   549	// ServerKeyID returns the server's GPG public key ID, in its long (16 capital
   550	// hex digits) format. If the server isn't running a sign handler, the error
   551	// will be ErrNoSigning.
   552	func (c *Client) ServerKeyID() (string, error) {
   553		if err := c.condDiscovery(); err != nil {
   554			return "", err
   555		}
   556		if c.serverKeyID == "" {
   557			return "", ErrNoSigning
   558		}
   559		return c.serverKeyID, nil
   560	}
   561	
   562	// ServerPublicKeyBlobRef returns the server's public key blobRef
   563	// If the server isn't running a sign handler, the error will be ErrNoSigning.
   564	func (c *Client) ServerPublicKeyBlobRef() (blob.Ref, error) {
   565		if err := c.condDiscovery(); err != nil {
   566			return blob.Ref{}, err
   567		}
   568	
   569		if !c.serverPublicKeyBlobRef.Valid() {
   570			return blob.Ref{}, ErrNoSigning
   571		}
   572		return c.serverPublicKeyBlobRef, nil
   573	}
   574	
   575	// SearchRoot returns the server's search handler.
   576	// If the server isn't running an index and search handler, the error
   577	// will be ErrNoSearchRoot.
   578	func (c *Client) SearchRoot() (string, error) {
   579		if err := c.condDiscovery(); err != nil {
   580			return "", err
   581		}
   582		if c.searchRoot == "" {
   583			return "", ErrNoSearchRoot
   584		}
   585		return c.searchRoot, nil
   586	}
   587	
   588	// HelpRoot returns the server's help handler.
   589	// If the server isn't running a help handler, the error will be
   590	// ErrNoHelpRoot.
   591	func (c *Client) HelpRoot() (string, error) {
   592		if err := c.condDiscovery(); err != nil {
   593			return "", err
   594		}
   595		if c.helpRoot == "" {
   596			return "", ErrNoHelpRoot
   597		}
   598		return c.helpRoot, nil
   599	}
   600	
   601	// ShareRoot returns the server's share handler prefix URL.
   602	// If the server isn't running a share handler, the error will be
   603	// ErrNoShareRoot.
   604	func (c *Client) ShareRoot() (string, error) {
   605		if err := c.condDiscovery(); err != nil {
   606			return "", err
   607		}
   608		if c.shareRoot == "" {
   609			return "", ErrNoShareRoot
   610		}
   611		return c.shareRoot, nil
   612	}
   613	
   614	// SignHandler returns the server's sign handler.
   615	// If the server isn't running a sign handler, the error will be
   616	// ErrNoSigning.
   617	func (c *Client) SignHandler() (string, error) {
   618		if err := c.condDiscovery(); err != nil {
   619			return "", err
   620		}
   621		if c.signHandler == "" {
   622			return "", ErrNoSigning
   623		}
   624		return c.signHandler, nil
   625	}
   626	
   627	// StorageGeneration returns the server's unique ID for its storage
   628	// generation, reset whenever storage is reset, moved, or partially
   629	// lost.
   630	//
   631	// This is a value that can be used in client cache keys to add
   632	// certainty that they're talking to the same instance as previously.
   633	//
   634	// If the server doesn't return such a value, the error will be
   635	// ErrNoStorageGeneration.
   636	func (c *Client) StorageGeneration() (string, error) {
   637		if err := c.condDiscovery(); err != nil {
   638			return "", err
   639		}
   640		if c.storageGen == "" {
   641			return "", ErrNoStorageGeneration
   642		}
   643		return c.storageGen, nil
   644	}
   645	
   646	// HasLegacySHA1 reports whether the server has SHA-1 blobs indexed.
   647	func (c *Client) HasLegacySHA1() (bool, error) {
   648		if err := c.condDiscovery(); err != nil {
   649			return false, err
   650		}
   651		return c.hasLegacySHA1, nil
   652	}
   653	
   654	// SyncInfo holds the data that were acquired with a discovery
   655	// and that are relevant to a syncHandler.
   656	type SyncInfo struct {
   657		From    string
   658		To      string
   659		ToIndex bool // whether this sync is from a blob storage to an index
   660	}
   661	
   662	// SyncHandlers returns the server's sync handlers "from" and
   663	// "to" prefix URLs.
   664	// If the server isn't running any sync handler, the error
   665	// will be ErrNoSync.
   666	func (c *Client) SyncHandlers() ([]*SyncInfo, error) {
   667		if err := c.condDiscovery(); err != nil {
   668			return nil, err
   669		}
   670		if c.syncHandlers == nil {
   671			return nil, ErrNoSync
   672		}
   673		return c.syncHandlers, nil
   674	}
   675	
   676	var _ search.GetRecentPermanoder = (*Client)(nil)
   677	
   678	// GetRecentPermanodes implements search.GetRecentPermanoder against a remote server over HTTP.
   679	func (c *Client) GetRecentPermanodes(ctx context.Context, req *search.RecentRequest) (*search.RecentResponse, error) {
   680		sr, err := c.SearchRoot()
   681		if err != nil {
   682			return nil, err
   683		}
   684		url := sr + req.URLSuffix()
   685		hreq := c.newRequest(ctx, "GET", url)
   686		hres, err := c.expect2XX(hreq)
   687		if err != nil {
   688			return nil, err
   689		}
   690		res := new(search.RecentResponse)
   691		if err := httputil.DecodeJSON(hres, res); err != nil {
   692			return nil, err
   693		}
   694		if err := res.Err(); err != nil {
   695			return nil, err
   696		}
   697		return res, nil
   698	}
   699	
   700	// GetPermanodesWithAttr searches for permanodes that match the given search request.
   701	// The Fuzzy option in the request must not be set, and the Attribute option
   702	// must be set.
   703	// Only indexed attributes may be queried.
   704	func (c *Client) GetPermanodesWithAttr(ctx context.Context, req *search.WithAttrRequest) (*search.WithAttrResponse, error) {
   705		sr, err := c.SearchRoot()
   706		if err != nil {
   707			return nil, err
   708		}
   709		url := sr + req.URLSuffix()
   710		hreq := c.newRequest(ctx, "GET", url)
   711		hres, err := c.expect2XX(hreq)
   712		if err != nil {
   713			return nil, err
   714		}
   715		res := new(search.WithAttrResponse)
   716		if err := httputil.DecodeJSON(hres, res); err != nil {
   717			return nil, err
   718		}
   719		if err := res.Err(); err != nil {
   720			return nil, err
   721		}
   722		return res, nil
   723	}
   724	
   725	func (c *Client) Describe(ctx context.Context, req *search.DescribeRequest) (*search.DescribeResponse, error) {
   726		sr, err := c.SearchRoot()
   727		if err != nil {
   728			return nil, err
   729		}
   730		url := sr + req.URLSuffixPost()
   731		body, err := json.MarshalIndent(req, "", "\t")
   732		if err != nil {
   733			return nil, err
   734		}
   735		hreq := c.newRequest(ctx, "POST", url, bytes.NewReader(body))
   736		hres, err := c.expect2XX(hreq)
   737		if err != nil {
   738			return nil, err
   739		}
   740		res := new(search.DescribeResponse)
   741		if err := httputil.DecodeJSON(hres, res); err != nil {
   742			return nil, err
   743		}
   744		return res, nil
   745	}
   746	
   747	func (c *Client) GetClaims(ctx context.Context, req *search.ClaimsRequest) (*search.ClaimsResponse, error) {
   748		sr, err := c.SearchRoot()
   749		if err != nil {
   750			return nil, err
   751		}
   752		url := sr + req.URLSuffix()
   753		hreq := c.newRequest(ctx, "GET", url)
   754		hres, err := c.expect2XX(hreq)
   755		if err != nil {
   756			return nil, err
   757		}
   758		res := new(search.ClaimsResponse)
   759		if err := httputil.DecodeJSON(hres, res); err != nil {
   760			return nil, err
   761		}
   762		return res, nil
   763	}
   764	
   765	func (c *Client) query(ctx context.Context, req *search.SearchQuery) (*http.Response, error) {
   766		sr, err := c.SearchRoot()
   767		if err != nil {
   768			return nil, err
   769		}
   770		url := sr + req.URLSuffix()
   771		body, err := json.Marshal(req)
   772		if err != nil {
   773			return nil, err
   774		}
   775		hreq := c.newRequest(ctx, "POST", url, bytes.NewReader(body))
   776		return c.expect2XX(hreq)
   777	}
   778	
   779	func (c *Client) Query(ctx context.Context, req *search.SearchQuery) (*search.SearchResult, error) {
   780		hres, err := c.query(ctx, req)
   781		if err != nil {
   782			return nil, err
   783		}
   784		res := new(search.SearchResult)
   785		if err := httputil.DecodeJSON(hres, res); err != nil {
   786			return nil, err
   787		}
   788		return res, nil
   789	}
   790	
   791	// QueryRaw sends req and returns the body of the response, which should be the
   792	// unparsed JSON of a search.SearchResult.
   793	func (c *Client) QueryRaw(ctx context.Context, req *search.SearchQuery) ([]byte, error) {
   794		hres, err := c.query(ctx, req)
   795		if err != nil {
   796			return nil, err
   797		}
   798		defer hres.Body.Close()
   799		return io.ReadAll(hres.Body)
   800	}
   801	
   802	// SearchExistingFileSchema does a search query looking for an
   803	// existing file with entire contents of wholeRef, then does a HEAD
   804	// request to verify the file still exists on the server. If so,
   805	// it returns that file schema's blobref.
   806	//
   807	// If multiple wholeRef values are provided, any may match. This is
   808	// used for searching for the file by multiple wholeRef hashes
   809	// (e.g. SHA-224 and SHA-1).
   810	//
   811	// It returns (zero, nil) if not found. A non-nil error is only returned
   812	// if there were problems searching.
   813	func (c *Client) SearchExistingFileSchema(ctx context.Context, wholeRef ...blob.Ref) (blob.Ref, error) {
   814		sr, err := c.SearchRoot()
   815		if err != nil {
   816			return blob.Ref{}, err
   817		}
   818		if len(wholeRef) == 0 {
   819			return blob.Ref{}, nil
   820		}
   821		url := sr + "camli/search/files"
   822		for i, ref := range wholeRef {
   823			if i == 0 {
   824				url += "?wholedigest=" + ref.String()
   825			} else {
   826				url += "&wholedigest=" + ref.String()
   827			}
   828		}
   829		req := c.newRequest(ctx, "GET", url)
   830		res, err := c.doReqGated(req)
   831		if err != nil {
   832			return blob.Ref{}, err
   833		}
   834		if res.StatusCode != 200 {
   835			body, _ := io.ReadAll(io.LimitReader(res.Body, 1<<20))
   836			res.Body.Close()
   837			return blob.Ref{}, fmt.Errorf("client: got status code %d from URL %s; body %s", res.StatusCode, url, body)
   838		}
   839		var ress camtypes.FileSearchResponse
   840		if err := httputil.DecodeJSON(res, &ress); err != nil {
   841			// Check that we're not just hitting the change introduced in 2018-01-13-6e8a5930c9fee81640c6c75a9a549fec98064186
   842			mismatch, err := c.versionMismatch(ctx)
   843			if err != nil {
   844				log.Printf("Could not verify whether client is too recent or server is too old: %v", err)
   845			} else if mismatch {
   846				return blob.Ref{}, fmt.Errorf("Client is too recent for this server. Use a client built before 2018-01-13-6e8a5930c9, or upgrade the server to after that revision.")
   847			}
   848			return blob.Ref{}, fmt.Errorf("client: error parsing JSON from URL %s: %v", url, err)
   849		}
   850		if len(ress.Files) == 0 {
   851			return blob.Ref{}, nil
   852		}
   853		for wholeRef, files := range ress.Files {
   854			for _, f := range files {
   855				if c.FileHasContents(ctx, f, blob.MustParse(wholeRef)) {
   856					return f, nil
   857				}
   858			}
   859		}
   860		return blob.Ref{}, nil
   861	}
   862	
   863	// versionMismatch returns true if the server was built before 2018-01-13 and
   864	// the client was built at or after 2018-01-13.
   865	func (c *Client) versionMismatch(ctx context.Context) (bool, error) {
   866		const shortRFC3339 = "2006-01-02"
   867		version := buildinfo.GitInfo
   868		if version == "" {
   869			return false, errors.New("unknown client version")
   870		}
   871		version = version[:10] // keep only the date part
   872		clientDate, err := time.Parse(shortRFC3339, version)
   873		if err != nil {
   874			return false, fmt.Errorf("could not parse date from version %q: %v", version, err)
   875		}
   876		apiChangeDate, _ := time.Parse(shortRFC3339, "2018-01-13")
   877		if !clientDate.After(apiChangeDate) {
   878			// client is old enough, all good.
   879			return false, nil
   880		}
   881		url := c.discoRoot() + "/status/status.json"
   882		req := c.newRequest(ctx, "GET", url)
   883		res, err := c.doReqGated(req)
   884		if err != nil {
   885			return false, err
   886		}
   887		if res.StatusCode != 200 {
   888			body, _ := io.ReadAll(io.LimitReader(res.Body, 1<<20))
   889			res.Body.Close()
   890			return false, fmt.Errorf("got status code %d from URL %s; body %s", res.StatusCode, url, body)
   891		}
   892		var status struct {
   893			Version string `json:"version"`
   894		}
   895		if err := httputil.DecodeJSON(res, &status); err != nil {
   896			return false, fmt.Errorf("error parsing JSON from URL %s: %v", url, err)
   897		}
   898		serverVersion := status.Version[:10]
   899		serverDate, err := time.Parse(shortRFC3339, serverVersion)
   900		if err != nil {
   901			return false, fmt.Errorf("could not parse date from server version %q: %v", status.Version, err)
   902		}
   903		if serverDate.After(apiChangeDate) {
   904			// server is recent enough, all good.
   905			return false, nil
   906		}
   907		return true, nil
   908	}
   909	
   910	// FileHasContents returns true iff f refers to a "file" or "bytes" schema blob,
   911	// the server is configured with a "download helper", and the server responds
   912	// that all chunks of 'f' are available and match the digest of wholeRef.
   913	func (c *Client) FileHasContents(ctx context.Context, f, wholeRef blob.Ref) bool {
   914		if err := c.condDiscovery(); err != nil {
   915			return false
   916		}
   917		if c.downloadHelper == "" {
   918			return false
   919		}
   920		req := c.newRequest(ctx, "HEAD", c.downloadHelper+f.String()+"/?verifycontents="+wholeRef.String())
   921		res, err := c.expect2XX(req)
   922		if err != nil {
   923			log.Printf("download helper HEAD error: %v", err)
   924			return false
   925		}
   926		defer res.Body.Close()
   927		return res.Header.Get("X-Camli-Contents") == wholeRef.String()
   928	}
   929	
   930	// prefix returns the URL prefix before "/camli/", or before
   931	// the blobref hash in case of a share URL.
   932	// Examples: http://foo.com:3179/bs or http://foo.com:3179/share
   933	func (c *Client) prefix() (string, error) {
   934		if err := c.prefixOnce.Do(c.initPrefix); err != nil {
   935			return "", err
   936		}
   937		return c.prefixv, nil
   938	}
   939	
   940	// blobPrefix returns the URL prefix before the blobref hash.
   941	// Example: http://foo.com:3179/bs/camli or http://foo.com:3179/share
   942	func (c *Client) blobPrefix() (string, error) {
   943		pfx, err := c.prefix()
   944		if err != nil {
   945			return "", err
   946		}
   947		if !c.isSharePrefix {
   948			pfx += "/camli"
   949		}
   950		return pfx, nil
   951	}
   952	
   953	// discoRoot returns the user defined server for this client. It prepends "https://" if no scheme was specified.
   954	func (c *Client) discoRoot() string {
   955		s := c.server
   956		if c.sameOrigin {
   957			s = strings.TrimPrefix(s, "http://")
   958			s = strings.TrimPrefix(s, "https://")
   959			parts := strings.SplitN(s, "/", 1)
   960			if len(parts) < 2 {
   961				return "/"
   962			}
   963			return "/" + parts[1]
   964		}
   965		if !strings.HasPrefix(s, "http") {
   966			s = "https://" + s
   967		}
   968		return s
   969	}
   970	
   971	// initPrefix uses the user provided server URL to define the URL
   972	// prefix to the blobserver root. If the server URL has a path
   973	// component then it is directly used, otherwise the blobRoot
   974	// from the discovery is used as the path.
   975	func (c *Client) initPrefix() error {
   976		c.isSharePrefix = false
   977		root := c.discoRoot()
   978		u, err := url.Parse(root)
   979		if err != nil {
   980			return err
   981		}
   982		if len(u.Path) > 1 {
   983			c.prefixv = strings.TrimRight(root, "/")
   984			return nil
   985		}
   986		return c.condDiscovery()
   987	}
   988	
   989	func (c *Client) condDiscovery() error {
   990		if c.sto != nil {
   991			return errors.New("client not using HTTP")
   992		}
   993		return c.discoOnce.Do(c.doDiscovery)
   994	}
   995	
   996	// DiscoveryDoc returns the server's JSON discovery document.
   997	// This method exists purely for the "camtool discovery" command.
   998	// Clients shouldn't have to parse this themselves.
   999	func (c *Client) DiscoveryDoc(ctx context.Context) (io.Reader, error) {
  1000		res, err := c.discoveryResp(ctx)
  1001		if err != nil {
  1002			return nil, err
  1003		}
  1004		defer res.Body.Close()
  1005		const maxSize = 1 << 20
  1006		all, err := io.ReadAll(io.LimitReader(res.Body, maxSize+1))
  1007		if err != nil {
  1008			return nil, err
  1009		}
  1010		if len(all) > maxSize {
  1011			return nil, errors.New("discovery document oddly large")
  1012		}
  1013		if len(all) > 0 && all[len(all)-1] != '\n' {
  1014			all = append(all, '\n')
  1015		}
  1016		return bytes.NewReader(all), err
  1017	}
  1018	
  1019	// HTTPVersion reports the HTTP version in use, such as "HTTP/1.1" or "HTTP/2.0".
  1020	func (c *Client) HTTPVersion(ctx context.Context) (string, error) {
  1021		req := c.newRequest(ctx, "HEAD", c.discoRoot(), nil)
  1022		res, err := c.doReqGated(req)
  1023		if err != nil {
  1024			return "", err
  1025		}
  1026		return res.Proto, err
  1027	}
  1028	
  1029	func (c *Client) discoveryResp(ctx context.Context) (*http.Response, error) {
  1030		// If the path is just "" or "/", do discovery against
  1031		// the URL to see which path we should actually use.
  1032		req := c.newRequest(ctx, "GET", c.discoRoot(), nil)
  1033		req.Header.Set("Accept", "text/x-camli-configuration")
  1034		res, err := c.doReqGated(req)
  1035		if err != nil {
  1036			return nil, err
  1037		}
  1038		if res.StatusCode != 200 {
  1039			res.Body.Close()
  1040			errMsg := fmt.Sprintf("got status %q from blobserver URL %q during configuration discovery", res.Status, c.discoRoot())
  1041			if res.StatusCode == http.StatusUnauthorized && c.authErr != nil {
  1042				errMsg = fmt.Sprintf("%v. %v", c.authErr, errMsg)
  1043			}
  1044			return nil, errors.New(errMsg)
  1045		}
  1046		// TODO(bradfitz): little weird in retrospect that we request
  1047		// text/x-camli-configuration and expect to get back
  1048		// text/javascript.  Make them consistent.
  1049		if ct := res.Header.Get("Content-Type"); ct != "text/javascript" {
  1050			res.Body.Close()
  1051			return nil, fmt.Errorf("Blobserver returned unexpected type %q from discovery", ct)
  1052		}
  1053		return res, nil
  1054	}
  1055	
  1056	func (c *Client) doDiscovery() error {
  1057		ctx := context.TODO()
  1058		root, err := url.Parse(c.discoRoot())
  1059		if err != nil {
  1060			return err
  1061		}
  1062	
  1063		res, err := c.discoveryResp(ctx)
  1064		if err != nil {
  1065			return err
  1066		}
  1067	
  1068		var disco camtypes.Discovery
  1069		if err := httputil.DecodeJSON(res, &disco); err != nil {
  1070			return err
  1071		}
  1072	
  1073		if disco.SearchRoot == "" {
  1074			c.searchRoot = ""
  1075		} else {
  1076			u, err := root.Parse(disco.SearchRoot)
  1077			if err != nil {
  1078				return fmt.Errorf("client: invalid searchRoot %q; failed to resolve", disco.SearchRoot)
  1079			}
  1080			c.searchRoot = u.String()
  1081		}
  1082	
  1083		u, err := root.Parse(disco.HelpRoot)
  1084		if err != nil {
  1085			return fmt.Errorf("client: invalid helpRoot %q; failed to resolve", disco.HelpRoot)
  1086		}
  1087		c.helpRoot = u.String()
  1088	
  1089		u, err = root.Parse(disco.ShareRoot)
  1090		if err != nil {
  1091			return fmt.Errorf("client: invalid shareRoot %q; failed to resolve", disco.ShareRoot)
  1092		}
  1093		c.shareRoot = u.String()
  1094	
  1095		c.storageGen = disco.StorageGeneration
  1096		c.hasLegacySHA1 = disco.HasLegacySHA1Index
  1097	
  1098		u, err = root.Parse(disco.BlobRoot)
  1099		if err != nil {
  1100			return fmt.Errorf("client: error resolving blobRoot: %v", err)
  1101		}
  1102		c.prefixv = strings.TrimRight(u.String(), "/")
  1103	
  1104		if disco.UIDiscovery != nil {
  1105			u, err = root.Parse(disco.DownloadHelper)
  1106			if err != nil {
  1107				return fmt.Errorf("client: invalid downloadHelper %q; failed to resolve", disco.DownloadHelper)
  1108			}
  1109			c.downloadHelper = u.String()
  1110		}
  1111	
  1112		if disco.SyncHandlers != nil {
  1113			for _, v := range disco.SyncHandlers {
  1114				ufrom, err := root.Parse(v.From)
  1115				if err != nil {
  1116					return fmt.Errorf("client: invalid %q \"from\" sync; failed to resolve", v.From)
  1117				}
  1118				uto, err := root.Parse(v.To)
  1119				if err != nil {
  1120					return fmt.Errorf("client: invalid %q \"to\" sync; failed to resolve", v.To)
  1121				}
  1122				c.syncHandlers = append(c.syncHandlers, &SyncInfo{
  1123					From:    ufrom.String(),
  1124					To:      uto.String(),
  1125					ToIndex: v.ToIndex,
  1126				})
  1127			}
  1128		}
  1129	
  1130		if disco.Signing != nil {
  1131			c.serverKeyID = disco.Signing.PublicKeyID
  1132			c.serverPublicKeyBlobRef = disco.Signing.PublicKeyBlobRef
  1133			c.signHandler = disco.Signing.SignHandler
  1134		}
  1135		return nil
  1136	}
  1137	
  1138	// GetJSON sends a GET request to url, and unmarshals the returned
  1139	// JSON response into data. The URL's host must match the client's
  1140	// configured server.
  1141	func (c *Client) GetJSON(ctx context.Context, url string, data interface{}) error {
  1142		if !strings.HasPrefix(url, c.discoRoot()) {
  1143			return fmt.Errorf("wrong URL (%q) for this server", url)
  1144		}
  1145		hreq := c.newRequest(ctx, "GET", url)
  1146		resp, err := c.expect2XX(hreq)
  1147		if err != nil {
  1148			return err
  1149		}
  1150		return httputil.DecodeJSON(resp, data)
  1151	}
  1152	
  1153	// Post is like http://golang.org/pkg/net/http/#Client.Post
  1154	// but with implementation details like gated requests. The
  1155	// URL's host must match the client's configured server.
  1156	func (c *Client) Post(ctx context.Context, url string, bodyType string, body io.Reader) error {
  1157		resp, err := c.post(ctx, url, bodyType, body)
  1158		if err != nil {
  1159			return err
  1160		}
  1161		return resp.Body.Close()
  1162	}
  1163	
  1164	// Sign sends a request to the sign handler on server to sign the contents of r,
  1165	// and return them signed. It uses the same implementation details, such as gated
  1166	// requests, as Post.
  1167	func (c *Client) Sign(ctx context.Context, server string, r io.Reader) (signed []byte, err error) {
  1168		signHandler, err := c.SignHandler()
  1169		if err != nil {
  1170			return nil, err
  1171		}
  1172		signServer := strings.TrimSuffix(server, "/") + signHandler
  1173		resp, err := c.post(ctx, signServer, "application/x-www-form-urlencoded", r)
  1174		if err != nil {
  1175			return nil, err
  1176		}
  1177		defer resp.Body.Close()
  1178		return io.ReadAll(resp.Body)
  1179	}
  1180	
  1181	func (c *Client) post(ctx context.Context, url string, bodyType string, body io.Reader) (*http.Response, error) {
  1182		if !c.sameOrigin && !strings.HasPrefix(url, c.discoRoot()) {
  1183			return nil, fmt.Errorf("wrong URL (%q) for this server", url)
  1184		}
  1185		req := c.newRequest(ctx, "POST", url, body)
  1186		req.Header.Set("Content-Type", bodyType)
  1187		res, err := c.expect2XX(req)
  1188		if err != nil {
  1189			return nil, err
  1190		}
  1191		return res, nil
  1192	}
  1193	
  1194	// newRequest creates a request with the authentication header, and with the
  1195	// appropriate scheme and port in the case of self-signed TLS.
  1196	func (c *Client) newRequest(ctx context.Context, method, url string, body ...io.Reader) *http.Request {
  1197		var bodyR io.Reader
  1198		if len(body) > 0 {
  1199			bodyR = body[0]
  1200		}
  1201		if len(body) > 1 {
  1202			panic("too many body arguments")
  1203		}
  1204		req, err := http.NewRequest(method, url, bodyR)
  1205		if err != nil {
  1206			panic(err.Error())
  1207		}
  1208		// not done by http.NewRequest in Go 1.0:
  1209		if br, ok := bodyR.(*bytes.Reader); ok {
  1210			req.ContentLength = int64(br.Len())
  1211		}
  1212		c.authMode.AddAuthHeader(req)
  1213		return req.WithContext(ctx)
  1214	}
  1215	
  1216	// expect2XX will doReqGated and promote HTTP response codes outside of
  1217	// the 200-299 range to a non-nil error containing the response body.
  1218	func (c *Client) expect2XX(req *http.Request) (*http.Response, error) {
  1219		res, err := c.doReqGated(req)
  1220		if err == nil && (res.StatusCode < 200 || res.StatusCode > 299) {
  1221			buf := new(bytes.Buffer)
  1222			io.CopyN(buf, res.Body, 1<<20)
  1223			res.Body.Close()
  1224			return res, fmt.Errorf("client: got status code %d from URL %s; body %s", res.StatusCode, req.URL.String(), buf.String())
  1225		}
  1226		return res, err
  1227	}
  1228	
  1229	func (c *Client) doReqGated(req *http.Request) (*http.Response, error) {
  1230		c.httpGate.Start()
  1231		defer c.httpGate.Done()
  1232		return c.httpClient.Do(req)
  1233	}
  1234	
  1235	// DialFunc returns the adequate dial function when we're on android.
  1236	func (c *Client) DialFunc() func(network, addr string) (net.Conn, error) {
  1237		if c.useTLS() {
  1238			return nil
  1239		}
  1240		if android.IsChild() {
  1241			return func(network, addr string) (net.Conn, error) {
  1242				return android.Dial(network, addr)
  1243			}
  1244		}
  1245		return nil
  1246	}
  1247	
  1248	func (c *Client) http2DialTLSFunc() func(network, addr string, cfg *tls.Config) (net.Conn, error) {
  1249		trustedCerts := c.getTrustedCerts()
  1250		if !c.insecureAnyTLSCert && len(trustedCerts) == 0 {
  1251			// TLS with normal/full verification.
  1252			// nil means http2 uses its default dialer.
  1253			return nil
  1254		}
  1255		return func(network, addr string, cfg *tls.Config) (net.Conn, error) {
  1256			// we own cfg, so we can mutate it:
  1257			cfg.InsecureSkipVerify = true
  1258			conn, err := tls.Dial(network, addr, cfg)
  1259			if err != nil {
  1260				return nil, err
  1261			}
  1262			if c.insecureAnyTLSCert {
  1263				return conn, err
  1264			}
  1265			state := conn.ConnectionState()
  1266			if p := state.NegotiatedProtocol; p != http2.NextProtoTLS {
  1267				return nil, fmt.Errorf("http2: unexpected ALPN protocol %q; want %q", p, http2.NextProtoTLS)
  1268			}
  1269			if !state.NegotiatedProtocolIsMutual {
  1270				return nil, errors.New("http2: could not negotiate protocol mutually")
  1271			}
  1272			certs := state.PeerCertificates
  1273			if len(certs) < 1 {
  1274				return nil, fmt.Errorf("no TLS peer certificates from %s", addr)
  1275			}
  1276			sig := hashutil.SHA256Prefix(certs[0].Raw)
  1277			for _, v := range trustedCerts {
  1278				if v == sig {
  1279					return conn, nil
  1280				}
  1281			}
  1282			return nil, fmt.Errorf("TLS server at %v presented untrusted certificate (signature %q)", addr, sig)
  1283		}
  1284	}
  1285	
  1286	// DialTLSFunc returns the adequate dial function, when using SSL, depending on
  1287	// whether we're using insecure TLS (certificate verification is disabled), or we
  1288	// have some trusted certs, or we're on android.
  1289	// If the client's config has some trusted certs, the server's certificate will
  1290	// be checked against those in the config after the TLS handshake.
  1291	func (c *Client) DialTLSFunc() func(network, addr string) (net.Conn, error) {
  1292		if !c.useTLS() {
  1293			return nil
  1294		}
  1295		trustedCerts := c.getTrustedCerts()
  1296		var stdTLS bool
  1297		if !c.insecureAnyTLSCert && len(trustedCerts) == 0 {
  1298			// TLS with normal/full verification.
  1299			stdTLS = true
  1300			if !android.IsChild() {
  1301				// Not android, so let the stdlib deal with it
  1302				return nil
  1303			}
  1304		}
  1305	
  1306		return func(network, addr string) (net.Conn, error) {
  1307			var conn *tls.Conn
  1308			var err error
  1309			if android.IsChild() {
  1310				ac, err := android.Dial(network, addr)
  1311				if err != nil {
  1312					return nil, err
  1313				}
  1314				var tlsConfig *tls.Config
  1315				if stdTLS {
  1316					tlsConfig, err = android.TLSConfig()
  1317					if err != nil {
  1318						return nil, err
  1319					}
  1320				} else {
  1321					tlsConfig = &tls.Config{InsecureSkipVerify: true}
  1322				}
  1323				// Since we're doing the TLS handshake ourselves, we need to set the ServerName,
  1324				// in case the server uses SNI (as is the case if it's relying on Let's Encrypt,
  1325				// for example).
  1326				tlsConfig.ServerName = c.serverNameOfAddr(addr)
  1327				conn = tls.Client(ac, tlsConfig)
  1328				if err := conn.Handshake(); err != nil {
  1329					return nil, err
  1330				}
  1331				if stdTLS {
  1332					// Normal TLS verification succeeded and we do not have
  1333					// additional trusted certificate fingerprints to check for.
  1334					return conn, nil
  1335				}
  1336			} else {
  1337				conn, err = tls.Dial(network, addr, &tls.Config{InsecureSkipVerify: true})
  1338				if err != nil {
  1339					return nil, err
  1340				}
  1341			}
  1342			if c.insecureAnyTLSCert {
  1343				return conn, nil
  1344			}
  1345			certs := conn.ConnectionState().PeerCertificates
  1346			if len(certs) < 1 {
  1347				return nil, fmt.Errorf("no TLS peer certificates from %s", addr)
  1348			}
  1349			sig := hashutil.SHA256Prefix(certs[0].Raw)
  1350			for _, v := range trustedCerts {
  1351				if v == sig {
  1352					return conn, nil
  1353				}
  1354			}
  1355			return nil, fmt.Errorf("TLS server at %v presented untrusted certificate (signature %q)", addr, sig)
  1356		}
  1357	}
  1358	
  1359	// serverNameOfAddr returns the host part of addr, or the empty string if addr
  1360	// is not a valid address (see net.Dial). Additionally, if host is an IP literal,
  1361	// serverNameOfAddr returns the empty string.
  1362	func (c *Client) serverNameOfAddr(addr string) string {
  1363		serverName, _, err := net.SplitHostPort(addr)
  1364		if err != nil {
  1365			c.printf("could not get server name from address %q: %v", addr, err)
  1366			return ""
  1367		}
  1368		if ip := net.ParseIP(serverName); ip != nil {
  1369			return ""
  1370		}
  1371		return serverName
  1372	}
  1373	
  1374	// Signer returns the client's Signer, if any. The Signer signs JSON
  1375	// mutation claims.
  1376	func (c *Client) Signer() (*schema.Signer, error) {
  1377		c.signerOnce.Do(c.signerInit)
  1378		return c.signer, c.signerErr
  1379	}
  1380	
  1381	func (c *Client) signerInit() {
  1382		c.signer, c.signerErr = c.buildSigner()
  1383	}
  1384	
  1385	func (c *Client) buildSigner() (*schema.Signer, error) {
  1386		c.initSignerPublicKeyBlobrefOnce.Do(c.initSignerPublicKeyBlobref)
  1387		if !c.signerPublicKeyRef.Valid() {
  1388			return nil, camtypes.ErrClientNoPublicKey
  1389		}
  1390		return schema.NewSigner(c.signerPublicKeyRef, strings.NewReader(c.publicKeyArmored), c.SecretRingFile())
  1391	}
  1392	
  1393	// sigTime optionally specifies the signature time.
  1394	// If zero, the current time is used.
  1395	func (c *Client) signBlob(ctx context.Context, bb schema.Buildable, sigTime time.Time) (string, error) {
  1396		signer, err := c.Signer()
  1397		if err != nil {
  1398			return "", err
  1399		}
  1400		return bb.Builder().SignAt(ctx, signer, sigTime)
  1401	}
  1402	
  1403	// UploadPublicKey uploads the public key (if one is defined), so
  1404	// subsequent (likely synchronous) indexing of uploaded signed blobs
  1405	// will have access to the public key to verify it. In the normal
  1406	// case, the stat cache prevents this from doing anything anyway.
  1407	func (c *Client) UploadPublicKey(ctx context.Context) error {
  1408		sigRef := c.SignerPublicKeyBlobref()
  1409		if !sigRef.Valid() {
  1410			return nil
  1411		}
  1412		var err error
  1413		if _, keyUploaded := c.haveCache.StatBlobCache(sigRef); !keyUploaded {
  1414			_, err = c.uploadString(ctx, c.publicKeyArmored, false)
  1415		}
  1416		return err
  1417	}
  1418	
  1419	// checkMatchingKeys compares the client's and the server's keys and logs if they differ.
  1420	func (c *Client) checkMatchingKeys() {
  1421		serverKey, err := c.ServerKeyID()
  1422		if err != nil {
  1423			log.Printf("Warning: Could not obtain ther server's key id: %v", err)
  1424			return
  1425		}
  1426		if serverKey != c.signer.KeyIDLong() {
  1427			log.Printf("Warning: client (%s) and server (%s) keys differ.", c.signer.KeyIDLong(), serverKey)
  1428		}
  1429	}
  1430	
  1431	func (c *Client) UploadAndSignBlob(ctx context.Context, b schema.AnyBlob) (*PutResult, error) {
  1432		signed, err := c.signBlob(ctx, b.Blob(), time.Time{})
  1433		if err != nil {
  1434			return nil, err
  1435		}
  1436		c.checkMatchingKeys()
  1437		if err := c.UploadPublicKey(ctx); err != nil {
  1438			return nil, err
  1439		}
  1440		return c.uploadString(ctx, signed, false)
  1441	}
  1442	
  1443	func (c *Client) UploadBlob(ctx context.Context, b schema.AnyBlob) (*PutResult, error) {
  1444		// TODO(bradfitz): ask the blob for its own blobref, rather
  1445		// than changing the hash function with uploadString?
  1446		return c.uploadString(ctx, b.Blob().JSON(), true)
  1447	}
  1448	
  1449	func (c *Client) uploadString(ctx context.Context, s string, stat bool) (*PutResult, error) {
  1450		uh := NewUploadHandleFromString(s)
  1451		uh.SkipStat = !stat
  1452		return c.Upload(ctx, uh)
  1453	}
  1454	
  1455	func (c *Client) UploadNewPermanode(ctx context.Context) (*PutResult, error) {
  1456		unsigned := schema.NewUnsignedPermanode()
  1457		return c.UploadAndSignBlob(ctx, unsigned)
  1458	}
  1459	
  1460	func (c *Client) UploadPlannedPermanode(ctx context.Context, key string, sigTime time.Time) (*PutResult, error) {
  1461		unsigned := schema.NewPlannedPermanode(key)
  1462		signed, err := c.signBlob(ctx, unsigned, sigTime)
  1463		if err != nil {
  1464			return nil, err
  1465		}
  1466		c.checkMatchingKeys()
  1467		if err := c.UploadPublicKey(ctx); err != nil {
  1468			return nil, err
  1469		}
  1470		return c.uploadString(ctx, signed, true)
  1471	}
  1472	
  1473	// IsIgnoredFile returns whether the file at fullpath should be ignored by pk-put.
  1474	// The fullpath is checked against the ignoredFiles list, trying the following rules in this order:
  1475	// 1) star-suffix style matching (.e.g *.jpg).
  1476	// 2) Shell pattern match as done by http://golang.org/pkg/path/filepath/#Match
  1477	// 3) If the pattern is an absolute path to a directory, fullpath matches if it is that directory or a child of it.
  1478	// 4) If the pattern is a relative path, fullpath matches if it has pattern as a path component (i.e the pattern is a part of fullpath that fits exactly between two path separators).
  1479	func (c *Client) IsIgnoredFile(fullpath string) bool {
  1480		c.initIgnoredFilesOnce.Do(c.initIgnoredFiles)
  1481		return c.ignoreChecker(fullpath)
  1482	}
  1483	
  1484	// Close closes the client.
  1485	func (c *Client) Close() error {
  1486		if cl, ok := c.sto.(io.Closer); ok {
  1487			return cl.Close()
  1488		}
  1489		if c := c.HTTPClient(); c != nil {
  1490			switch t := c.Transport.(type) {
  1491			case *http.Transport:
  1492				t.CloseIdleConnections()
  1493			case *http2.Transport:
  1494				t.CloseIdleConnections()
  1495			}
  1496		}
  1497		return nil
  1498	}
  1499	
  1500	func httpGateSize(rt http.RoundTripper) int {
  1501		switch v := rt.(type) {
  1502		case *httputil.StatsTransport:
  1503			return httpGateSize(v.Transport)
  1504		case *http.Transport:
  1505			return maxParallelHTTP_h1
  1506		case *http2.Transport:
  1507			return maxParallelHTTP_h2
  1508		default:
  1509			return maxParallelHTTP_h1 // conservative default
  1510		}
  1511	}