1	/*
     2	Copyright 2011 The Perkeep Authors.
     3	
     4	Licensed under the Apache License, Version 2.0 (the "License");
     5	you may not use this file except in compliance with the License.
     6	You may obtain a copy of the License at
     7	
     8	     http://www.apache.org/licenses/LICENSE-2.0
     9	
    10	Unless required by applicable law or agreed to in writing, software
    11	distributed under the License is distributed on an "AS IS" BASIS,
    12	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13	See the License for the specific language governing permissions and
    14	limitations under the License.
    15	*/
    16	
    17	// Package client implements a Perkeep client.
    18	package client // import "perkeep.org/pkg/client"
    19	
    20	import (
    21		"bytes"
    22		"context"
    23		"crypto/tls"
    24		"encoding/json"
    25		"errors"
    26		"fmt"
    27		"io"
    28		"io/ioutil"
    29		"log"
    30		"net"
    31		"net/http"
    32		"net/url"
    33		"os"
    34		"regexp"
    35		"strings"
    36		"sync"
    37		"time"
    38	
    39		"perkeep.org/internal/hashutil"
    40		"perkeep.org/internal/httputil"
    41		"perkeep.org/internal/osutil"
    42		"perkeep.org/pkg/auth"
    43		"perkeep.org/pkg/blob"
    44		"perkeep.org/pkg/blobserver"
    45		"perkeep.org/pkg/buildinfo"
    46		"perkeep.org/pkg/client/android"
    47		"perkeep.org/pkg/schema"
    48		"perkeep.org/pkg/search"
    49		"perkeep.org/pkg/types/camtypes"
    50	
    51		"go4.org/syncutil"
    52		"golang.org/x/net/http2"
    53	)
    54	
    55	// A Client provides access to a Perkeep server.
    56	//
    57	// After use, a Client should be closed via its Close method to
    58	// release idle HTTP connections or other resourcedds.
    59	type Client struct {
    60		// server is the input from user, pre-discovery.
    61		// For example "http://foo.com" or "foo.com:1234".
    62		// It is the responsibility of initPrefix to parse
    63		// server and set prefix, including doing discovery
    64		// to figure out what the proper server-declared
    65		// prefix is.
    66		server string
    67	
    68		prefixOnce    syncutil.Once // guards init of following 2 fields
    69		prefixv       string        // URL prefix before "/camli/"
    70		isSharePrefix bool          // URL is a request for a share blob
    71	
    72		discoOnce              syncutil.Once
    73		searchRoot             string      // Handler prefix, or "" if none
    74		downloadHelper         string      // or "" if none
    75		storageGen             string      // storage generation, or "" if not reported
    76		hasLegacySHA1          bool        // Whether server has SHA-1 blobs indexed.
    77		syncHandlers           []*SyncInfo // "from" and "to" url prefix for each syncHandler
    78		serverKeyID            string      // Server's GPG public key ID.
    79		helpRoot               string      // Handler prefix, or "" if none
    80		shareRoot              string      // Share handler prefix, or "" if none
    81		serverPublicKeyBlobRef blob.Ref    // Server's public key blobRef
    82	
    83		signerOnce  sync.Once
    84		signer      *schema.Signer
    85		signerErr   error
    86		signHandler string // Handler prefix, or "" if none
    87	
    88		authMode auth.AuthMode
    89		// authErr is set when no auth config is found but we want to defer warning
    90		// until discovery fails.
    91		authErr error
    92	
    93		httpClient *http.Client
    94		haveCache  HaveCache
    95	
    96		// If sto is set, it's used before the httpClient or other network operations.
    97		sto blobserver.Storage
    98	
    99		initTrustedCertsOnce sync.Once
   100	
   101		// We define a certificate fingerprint as the 20 digits lowercase prefix
   102		// of the SHA256 of the complete certificate (in ASN.1 DER encoding).
   103		// trustedCerts contains the fingerprints of the self-signed
   104		// certificates we trust.
   105		// If not empty, (and if using TLS) the full x509 verification is
   106		// disabled, and we instead check the server's certificate against
   107		// this list.
   108		// The perkeepd server prints the fingerprint to add to the config
   109		// when starting.
   110		trustedCerts []string
   111	
   112		// insecureAnyTLSCert disables all TLS cert checking,
   113		// including the trustedCerts field above.
   114		insecureAnyTLSCert bool
   115	
   116		initIgnoredFilesOnce sync.Once
   117		// list of files that pk-put should ignore.
   118		// Defaults to empty, but pk-put init creates a config with a non
   119		// empty list.
   120		// See IsIgnoredFile for the matching rules.
   121		ignoredFiles  []string
   122		ignoreChecker func(path string) bool
   123	
   124		initSignerPublicKeyBlobrefOnce sync.Once
   125		signerPublicKeyRef             blob.Ref
   126		publicKeyArmored               string
   127	
   128		statsMutex sync.Mutex
   129		stats      Stats
   130	
   131		// via maps the access path from a share root to a desired target.
   132		// It is non-nil when in "sharing" mode, where the Client is fetching
   133		// a share.
   134		viaMu sync.RWMutex
   135		via   map[blob.Ref]blob.Ref // target => via (target is referenced from via)
   136	
   137		// Verbose controls how much logging from the client is printed. The caller
   138		// should set it only before using the client, and treat it as read-only after
   139		// that.
   140		Verbose bool
   141		// Logger is the logger used by the client. It defaults to a standard
   142		// logger to os.Stderr if the client is initialized by one of the package's
   143		// functions. Like Verbose, it should be set only before using the client, and
   144		// not be modified afterwards.
   145		Logger *log.Logger
   146	
   147		httpGate        *syncutil.Gate
   148		transportConfig *TransportConfig // or nil
   149	
   150		noExtConfig bool // no external config; config file and env vars are ignored.
   151	
   152		// sameOrigin indicates whether URLs in requests should be stripped from
   153		// their scheme and HostPort parts. This is meant for when using the client
   154		// through gopherjs in the web UI. Because we'll run into CORS errors if
   155		// requests have a Host part.
   156		sameOrigin bool
   157	}
   158	
   159	const maxParallelHTTP_h1 = 5
   160	const maxParallelHTTP_h2 = 50
   161	
   162	// inGopherJS reports whether the client package is compiled by GopherJS, for use
   163	// in the browser.
   164	var inGopherJS bool
   165	
   166	// New returns a new Perkeep Client.
   167	//
   168	// By default, with no options, it uses the client as configured in
   169	// the environment or default configuration files.
   170	func New(opts ...ClientOption) (*Client, error) {
   171		c := &Client{
   172			haveCache: noHaveCache{},
   173			Logger:    log.New(os.Stderr, "", log.Ldate|log.Ltime),
   174			authMode:  auth.None{},
   175		}
   176		for _, v := range opts {
   177			v.modifyClient(c)
   178		}
   179		if c.sto != nil && len(opts) > 1 {
   180			return nil, errors.New("use of OptionUseStorageClient precludes use of any other options")
   181		}
   182	
   183		if inGopherJS {
   184			c.noExtConfig = true
   185			c.sameOrigin = true
   186		}
   187		if c.noExtConfig {
   188			c.setDefaultHTTPClient()
   189			return c, nil
   190		}
   191	
   192		if c.server != "" {
   193			if !isURLOrHostPort(c.server) {
   194				configOnce.Do(parseConfig)
   195				serverConf, ok := config.Servers[c.server]
   196				if !ok {
   197					log.Fatalf("%q looks like a server alias, but no such alias found in config at %v", c.server, osutil.UserClientConfigPath())
   198				}
   199				c.server = serverConf.Server
   200			}
   201			c.setDefaultHTTPClient()
   202			return c, nil
   203		}
   204	
   205		var err error
   206		c.server, err = getServer()
   207		if err != nil {
   208			return nil, err
   209		}
   210		err = c.SetupAuth()
   211		if err != nil {
   212			return nil, err
   213		}
   214		c.setDefaultHTTPClient()
   215		return c, nil
   216	}
   217	
   218	func (c *Client) setDefaultHTTPClient() {
   219		if c.httpClient == nil {
   220			c.httpClient = &http.Client{
   221				Transport: c.transportForConfig(c.transportConfig),
   222			}
   223		}
   224		c.httpGate = syncutil.NewGate(httpGateSize(c.httpClient.Transport))
   225	}
   226	
   227	// NewOrFail is like New, but calls log.Fatal instead of returning an error.
   228	func NewOrFail(opts ...ClientOption) *Client {
   229		c, err := New(opts...)
   230		if err != nil {
   231			log.Fatalf("error creating client: %v", err)
   232		}
   233		return c
   234	}
   235	
   236	// NewPathClient returns a new client accessing a subpath of c.
   237	func (c *Client) NewPathClient(path string) (*Client, error) {
   238		u, err := url.Parse(c.server)
   239		if err != nil {
   240			return nil, fmt.Errorf("bogus server %q for NewPathClient receiver: %v", c.server, err)
   241		}
   242		u.Path = path
   243		pc, err := New(OptionServer(u.String()))
   244		if err != nil {
   245			return nil, err
   246		}
   247		pc.authMode = c.authMode
   248		pc.discoOnce.Do(noop)
   249		return pc, nil
   250	}
   251	
   252	// TransportConfig contains options for how HTTP requests are made.
   253	type TransportConfig struct {
   254		// Proxy optionally specifies the Proxy for the transport. Useful with
   255		// pk-put for debugging even localhost requests.
   256		Proxy   func(*http.Request) (*url.URL, error)
   257		Verbose bool // Verbose enables verbose logging of HTTP requests.
   258	}
   259	
   260	func (c *Client) useHTTP2(tc *TransportConfig) bool {
   261		if !c.useTLS() {
   262			return false
   263		}
   264		if android.IsChild() {
   265			// No particular reason; just untested so far.
   266			return false
   267		}
   268		if os.Getenv("HTTPS_PROXY") != "" || os.Getenv("https_proxy") != "" ||
   269			(tc != nil && tc.Proxy != nil) {
   270			// Also just untested. Which proxies support h2 anyway?
   271			return false
   272		}
   273		return true
   274	}
   275	
   276	// transportForConfig returns a transport for the client, setting the correct
   277	// Proxy, Dial, and TLSClientConfig if needed. It does not mutate c.
   278	// It is the caller's responsibility to then use that transport to set
   279	// the client's httpClient with SetHTTPClient.
   280	func (c *Client) transportForConfig(tc *TransportConfig) http.RoundTripper {
   281		if inGopherJS {
   282			// Calls to net.Dial* functions - which would happen if the client's transport
   283			// is not nil - are prohibited with GopherJS. So we force nil here, so that the
   284			// call to transportForConfig in newClient is of no consequence when on the
   285			// browser.
   286			return nil
   287		}
   288		if c == nil {
   289			return nil
   290		}
   291		var transport http.RoundTripper
   292		proxy := http.ProxyFromEnvironment
   293		if tc != nil && tc.Proxy != nil {
   294			proxy = tc.Proxy
   295		}
   296	
   297		if c.useHTTP2(tc) {
   298			transport = &http2.Transport{
   299				DialTLS: c.http2DialTLSFunc(),
   300			}
   301		} else {
   302			transport = &http.Transport{
   303				DialTLS:             c.DialTLSFunc(),
   304				Dial:                c.DialFunc(),
   305				Proxy:               proxy,
   306				MaxIdleConnsPerHost: maxParallelHTTP_h1,
   307			}
   308		}
   309		httpStats := &httputil.StatsTransport{
   310			Transport: transport,
   311		}
   312		if tc != nil {
   313			httpStats.VerboseLog = tc.Verbose
   314		}
   315		transport = httpStats
   316		if android.IsChild() {
   317			transport = &android.StatsTransport{Rt: transport}
   318		}
   319		return transport
   320	}
   321	
   322	// HTTPStats returns the client's underlying httputil.StatsTransport, if in use.
   323	// If another transport is being used, nil is returned.
   324	func (c *Client) HTTPStats() *httputil.StatsTransport {
   325		st, _ := c.httpClient.Transport.(*httputil.StatsTransport)
   326		return st
   327	}
   328	
   329	type ClientOption interface {
   330		modifyClient(*Client)
   331	}
   332	
   333	// OptionServer returns a Client constructor option that forces use of
   334	// the provided server.
   335	//
   336	// The provided server is either "host:port" (assumed http, not https)
   337	// or a URL prefix, with or without a path, or a server alias from the
   338	// client configuration file. A server alias should not be confused
   339	// with a hostname, therefore it cannot contain any colon or period.
   340	func OptionServer(server string) ClientOption {
   341		return optionServer(server)
   342	}
   343	
   344	type optionServer string
   345	
   346	func (s optionServer) modifyClient(c *Client) { c.server = string(s) }
   347	
   348	// OptionUseStorageClient returns a Client constructor option that
   349	// forces use of the provided blob storage target.
   350	//
   351	// This exists mainly so all the convenience methods on
   352	// Client (e.g. the Upload variants) are available against storage
   353	// directly.
   354	//
   355	// Use of OptionUseStorageClient is mutually exclusively
   356	// with all other options, although it does imply
   357	// OptionNoExternalConfig(true).
   358	func OptionUseStorageClient(s blobserver.Storage) ClientOption {
   359		return optionStorage{s}
   360	}
   361	
   362	type optionStorage struct {
   363		sto blobserver.Storage
   364	}
   365	
   366	func (o optionStorage) modifyClient(c *Client) {
   367		c.sto = o.sto
   368	}
   369	
   370	// OptionTransportConfig returns a ClientOption that makes the client use
   371	// the provided transport configuration options.
   372	func OptionTransportConfig(tc *TransportConfig) ClientOption {
   373		return optionTransportConfig{tc}
   374	}
   375	
   376	type optionTransportConfig struct {
   377		tc *TransportConfig
   378	}
   379	
   380	func (o optionTransportConfig) modifyClient(c *Client) { c.transportConfig = o.tc }
   381	
   382	// OptionInsecure returns a ClientOption that controls whether HTTP
   383	// requests are allowed to be insecure (over HTTP or HTTPS without TLS
   384	// certificate checking). Use of this is strongly discouraged except
   385	// for local testing.
   386	func OptionInsecure(v bool) ClientOption {
   387		return optionInsecure(v)
   388	}
   389	
   390	type optionInsecure bool
   391	
   392	func (o optionInsecure) modifyClient(c *Client) { c.insecureAnyTLSCert = bool(o) }
   393	
   394	// OptionTrustedCert returns a ClientOption that makes the client
   395	// trust the provide self-signed cert signature. The value should be
   396	// the 20 byte hex prefix of the SHA-256 of the cert, as printed by
   397	// the perkeepd server on start-up.
   398	//
   399	// If cert is empty, the option has no effect.
   400	func OptionTrustedCert(cert string) ClientOption {
   401		// TODO: remove this whole function now that we have LetsEncrypt?
   402		return optionTrustedCert(cert)
   403	}
   404	
   405	type optionTrustedCert string
   406	
   407	func (o optionTrustedCert) modifyClient(c *Client) {
   408		cert := string(o)
   409		if cert != "" {
   410			c.initTrustedCertsOnce.Do(func() {})
   411			c.trustedCerts = []string{string(o)}
   412		}
   413	}
   414	
   415	type optionNoExtConfig bool
   416	
   417	func (o optionNoExtConfig) modifyClient(c *Client) { c.noExtConfig = bool(o) }
   418	
   419	// OptionNoExternalConfig returns a Client constructor option that
   420	// prevents any on-disk config files or environment variables from
   421	// influencing the client. It may still use the disk for caches.
   422	func OptionNoExternalConfig() ClientOption {
   423		return optionNoExtConfig(true)
   424	}
   425	
   426	type optionAuthMode struct {
   427		m auth.AuthMode
   428	}
   429	
   430	func (o optionAuthMode) modifyClient(c *Client) { c.authMode = o.m }
   431	
   432	// OptionAuthMode returns a Client constructor option that sets the
   433	// client's authentication mode.
   434	func OptionAuthMode(m auth.AuthMode) ClientOption {
   435		return optionAuthMode{m}
   436	}
   437	
   438	// noop is for use with syncutil.Onces.
   439	func noop() error { return nil }
   440	
   441	var shareURLRx = regexp.MustCompile(`^(.+)/(` + blob.Pattern + ")$")
   442	
   443	// NewFromShareRoot uses shareBlobURL to set up and return a client that
   444	// will be used to fetch shared blobs.
   445	func NewFromShareRoot(ctx context.Context, shareBlobURL string, opts ...ClientOption) (c *Client, target blob.Ref, err error) {
   446		var root string
   447		m := shareURLRx.FindStringSubmatch(shareBlobURL)
   448		if m == nil {
   449			return nil, blob.Ref{}, fmt.Errorf("Unknown share URL base")
   450		}
   451		c, err = New(append(opts[:len(opts):cap(opts)], OptionServer(m[1]))...)
   452		if err != nil {
   453			return nil, blob.Ref{}, err
   454		}
   455		c.discoOnce.Do(noop)
   456		c.prefixOnce.Do(noop)
   457		c.prefixv = m[1]
   458		c.isSharePrefix = true
   459		c.authMode = auth.None{}
   460		c.via = make(map[blob.Ref]blob.Ref)
   461		root = m[2]
   462	
   463		req := c.newRequest(ctx, "GET", shareBlobURL, nil)
   464		res, err := c.expect2XX(req)
   465		if err != nil {
   466			return nil, blob.Ref{}, fmt.Errorf("error fetching %s: %v", shareBlobURL, err)
   467		}
   468		defer res.Body.Close()
   469		var buf bytes.Buffer
   470		rootbr, ok := blob.Parse(root)
   471		if !ok {
   472			return nil, blob.Ref{}, fmt.Errorf("invalid root blob ref for sharing: %q", root)
   473		}
   474		b, err := schema.BlobFromReader(rootbr, io.TeeReader(res.Body, &buf))
   475		if err != nil {
   476			return nil, blob.Ref{}, fmt.Errorf("error parsing JSON from %s: %v , with response: %q", shareBlobURL, err, buf.Bytes())
   477		}
   478		if b.ShareAuthType() != schema.ShareHaveRef {
   479			return nil, blob.Ref{}, fmt.Errorf("unknown share authType of %q", b.ShareAuthType())
   480		}
   481		target = b.ShareTarget()
   482		if !target.Valid() {
   483			return nil, blob.Ref{}, fmt.Errorf("no target")
   484		}
   485		c.via[target] = rootbr
   486		return c, target, nil
   487	}
   488	
   489	// SetHTTPClient sets the Perkeep client's HTTP client.
   490	// If nil, the default HTTP client is used.
   491	func (c *Client) SetHTTPClient(client *http.Client) {
   492		if client == nil {
   493			client = http.DefaultClient
   494		}
   495		c.httpClient = client
   496	}
   497	
   498	// HTTPClient returns the Client's underlying http.Client.
   499	func (c *Client) HTTPClient() *http.Client {
   500		return c.httpClient
   501	}
   502	
   503	// A HaveCache caches whether a remote blobserver has a blob.
   504	type HaveCache interface {
   505		StatBlobCache(br blob.Ref) (size uint32, ok bool)
   506		NoteBlobExists(br blob.Ref, size uint32)
   507	}
   508	
   509	type noHaveCache struct{}
   510	
   511	func (noHaveCache) StatBlobCache(blob.Ref) (uint32, bool) { return 0, false }
   512	func (noHaveCache) NoteBlobExists(blob.Ref, uint32)       {}
   513	
   514	func (c *Client) SetHaveCache(cache HaveCache) {
   515		if cache == nil {
   516			cache = noHaveCache{}
   517		}
   518		c.haveCache = cache
   519	}
   520	
   521	func (c *Client) printf(format string, v ...interface{}) {
   522		if c.Verbose && c.Logger != nil {
   523			c.Logger.Printf(format, v...)
   524		}
   525	}
   526	
   527	func (c *Client) Stats() Stats {
   528		c.statsMutex.Lock()
   529		defer c.statsMutex.Unlock()
   530		return c.stats // copy
   531	}
   532	
   533	// ErrNoSearchRoot is returned by SearchRoot if the server doesn't support search.
   534	var ErrNoSearchRoot = errors.New("client: server doesn't support search")
   535	
   536	// ErrNoHelpRoot is returned by HelpRoot if the server doesn't have a help handler.
   537	var ErrNoHelpRoot = errors.New("client: server does not have a help handler")
   538	
   539	// ErrNoShareRoot is returned by ShareRoot if the server doesn't have a share handler.
   540	var ErrNoShareRoot = errors.New("client: server does not have a share handler")
   541	
   542	// ErrNoSigning is returned by ServerKeyID if the server doesn't support signing.
   543	var ErrNoSigning = fmt.Errorf("client: server doesn't support signing")
   544	
   545	// ErrNoStorageGeneration is returned by StorageGeneration if the
   546	// server doesn't report a storage generation value.
   547	var ErrNoStorageGeneration = errors.New("client: server doesn't report a storage generation")
   548	
   549	// ErrNoSync is returned by SyncHandlers if the server does not advertise syncs.
   550	var ErrNoSync = errors.New("client: server has no sync handlers")
   551	
   552	// BlobRoot returns the server's blobroot URL prefix.
   553	// If the client was constructed with an explicit path,
   554	// that path is used. Otherwise the server's
   555	// default advertised blobRoot is used.
   556	func (c *Client) BlobRoot() (string, error) {
   557		prefix, err := c.prefix()
   558		if err != nil {
   559			return "", err
   560		}
   561		return prefix + "/", nil
   562	}
   563	
   564	// ServerKeyID returns the server's GPG public key ID, in its long (16 capital
   565	// hex digits) format. If the server isn't running a sign handler, the error
   566	// will be ErrNoSigning.
   567	func (c *Client) ServerKeyID() (string, error) {
   568		if err := c.condDiscovery(); err != nil {
   569			return "", err
   570		}
   571		if c.serverKeyID == "" {
   572			return "", ErrNoSigning
   573		}
   574		return c.serverKeyID, nil
   575	}
   576	
   577	// ServerPublicKeyBlobRef returns the server's public key blobRef
   578	// If the server isn't running a sign handler, the error will be ErrNoSigning.
   579	func (c *Client) ServerPublicKeyBlobRef() (blob.Ref, error) {
   580		if err := c.condDiscovery(); err != nil {
   581			return blob.Ref{}, err
   582		}
   583	
   584		if !c.serverPublicKeyBlobRef.Valid() {
   585			return blob.Ref{}, ErrNoSigning
   586		}
   587		return c.serverPublicKeyBlobRef, nil
   588	}
   589	
   590	// SearchRoot returns the server's search handler.
   591	// If the server isn't running an index and search handler, the error
   592	// will be ErrNoSearchRoot.
   593	func (c *Client) SearchRoot() (string, error) {
   594		if err := c.condDiscovery(); err != nil {
   595			return "", err
   596		}
   597		if c.searchRoot == "" {
   598			return "", ErrNoSearchRoot
   599		}
   600		return c.searchRoot, nil
   601	}
   602	
   603	// HelpRoot returns the server's help handler.
   604	// If the server isn't running a help handler, the error will be
   605	// ErrNoHelpRoot.
   606	func (c *Client) HelpRoot() (string, error) {
   607		if err := c.condDiscovery(); err != nil {
   608			return "", err
   609		}
   610		if c.helpRoot == "" {
   611			return "", ErrNoHelpRoot
   612		}
   613		return c.helpRoot, nil
   614	}
   615	
   616	// ShareRoot returns the server's share handler prefix URL.
   617	// If the server isn't running a share handler, the error will be
   618	// ErrNoShareRoot.
   619	func (c *Client) ShareRoot() (string, error) {
   620		if err := c.condDiscovery(); err != nil {
   621			return "", err
   622		}
   623		if c.shareRoot == "" {
   624			return "", ErrNoShareRoot
   625		}
   626		return c.shareRoot, nil
   627	}
   628	
   629	// SignHandler returns the server's sign handler.
   630	// If the server isn't running a sign handler, the error will be
   631	// ErrNoSigning.
   632	func (c *Client) SignHandler() (string, error) {
   633		if err := c.condDiscovery(); err != nil {
   634			return "", err
   635		}
   636		if c.signHandler == "" {
   637			return "", ErrNoSigning
   638		}
   639		return c.signHandler, nil
   640	}
   641	
   642	// StorageGeneration returns the server's unique ID for its storage
   643	// generation, reset whenever storage is reset, moved, or partially
   644	// lost.
   645	//
   646	// This is a value that can be used in client cache keys to add
   647	// certainty that they're talking to the same instance as previously.
   648	//
   649	// If the server doesn't return such a value, the error will be
   650	// ErrNoStorageGeneration.
   651	func (c *Client) StorageGeneration() (string, error) {
   652		if err := c.condDiscovery(); err != nil {
   653			return "", err
   654		}
   655		if c.storageGen == "" {
   656			return "", ErrNoStorageGeneration
   657		}
   658		return c.storageGen, nil
   659	}
   660	
   661	// HasLegacySHA1 reports whether the server has SHA-1 blobs indexed.
   662	func (c *Client) HasLegacySHA1() (bool, error) {
   663		if err := c.condDiscovery(); err != nil {
   664			return false, err
   665		}
   666		return c.hasLegacySHA1, nil
   667	}
   668	
   669	// SyncInfo holds the data that were acquired with a discovery
   670	// and that are relevant to a syncHandler.
   671	type SyncInfo struct {
   672		From    string
   673		To      string
   674		ToIndex bool // whether this sync is from a blob storage to an index
   675	}
   676	
   677	// SyncHandlers returns the server's sync handlers "from" and
   678	// "to" prefix URLs.
   679	// If the server isn't running any sync handler, the error
   680	// will be ErrNoSync.
   681	func (c *Client) SyncHandlers() ([]*SyncInfo, error) {
   682		if err := c.condDiscovery(); err != nil {
   683			return nil, err
   684		}
   685		if c.syncHandlers == nil {
   686			return nil, ErrNoSync
   687		}
   688		return c.syncHandlers, nil
   689	}
   690	
   691	var _ search.GetRecentPermanoder = (*Client)(nil)
   692	
   693	// GetRecentPermanodes implements search.GetRecentPermanoder against a remote server over HTTP.
   694	func (c *Client) GetRecentPermanodes(ctx context.Context, req *search.RecentRequest) (*search.RecentResponse, error) {
   695		sr, err := c.SearchRoot()
   696		if err != nil {
   697			return nil, err
   698		}
   699		url := sr + req.URLSuffix()
   700		hreq := c.newRequest(ctx, "GET", url)
   701		hres, err := c.expect2XX(hreq)
   702		if err != nil {
   703			return nil, err
   704		}
   705		res := new(search.RecentResponse)
   706		if err := httputil.DecodeJSON(hres, res); err != nil {
   707			return nil, err
   708		}
   709		if err := res.Err(); err != nil {
   710			return nil, err
   711		}
   712		return res, nil
   713	}
   714	
   715	// GetPermanodesWithAttr searches for permanodes that match the given search request.
   716	// The Fuzzy option in the request must not be set, and the Attribute option
   717	// must be set.
   718	// Only indexed attributes may be queried.
   719	func (c *Client) GetPermanodesWithAttr(ctx context.Context, req *search.WithAttrRequest) (*search.WithAttrResponse, error) {
   720		sr, err := c.SearchRoot()
   721		if err != nil {
   722			return nil, err
   723		}
   724		url := sr + req.URLSuffix()
   725		hreq := c.newRequest(ctx, "GET", url)
   726		hres, err := c.expect2XX(hreq)
   727		if err != nil {
   728			return nil, err
   729		}
   730		res := new(search.WithAttrResponse)
   731		if err := httputil.DecodeJSON(hres, res); err != nil {
   732			return nil, err
   733		}
   734		if err := res.Err(); err != nil {
   735			return nil, err
   736		}
   737		return res, nil
   738	}
   739	
   740	func (c *Client) Describe(ctx context.Context, req *search.DescribeRequest) (*search.DescribeResponse, error) {
   741		sr, err := c.SearchRoot()
   742		if err != nil {
   743			return nil, err
   744		}
   745		url := sr + req.URLSuffixPost()
   746		body, err := json.MarshalIndent(req, "", "\t")
   747		if err != nil {
   748			return nil, err
   749		}
   750		hreq := c.newRequest(ctx, "POST", url, bytes.NewReader(body))
   751		hres, err := c.expect2XX(hreq)
   752		if err != nil {
   753			return nil, err
   754		}
   755		res := new(search.DescribeResponse)
   756		if err := httputil.DecodeJSON(hres, res); err != nil {
   757			return nil, err
   758		}
   759		return res, nil
   760	}
   761	
   762	func (c *Client) GetClaims(ctx context.Context, req *search.ClaimsRequest) (*search.ClaimsResponse, error) {
   763		sr, err := c.SearchRoot()
   764		if err != nil {
   765			return nil, err
   766		}
   767		url := sr + req.URLSuffix()
   768		hreq := c.newRequest(ctx, "GET", url)
   769		hres, err := c.expect2XX(hreq)
   770		if err != nil {
   771			return nil, err
   772		}
   773		res := new(search.ClaimsResponse)
   774		if err := httputil.DecodeJSON(hres, res); err != nil {
   775			return nil, err
   776		}
   777		return res, nil
   778	}
   779	
   780	func (c *Client) query(ctx context.Context, req *search.SearchQuery) (*http.Response, error) {
   781		sr, err := c.SearchRoot()
   782		if err != nil {
   783			return nil, err
   784		}
   785		url := sr + req.URLSuffix()
   786		body, err := json.Marshal(req)
   787		if err != nil {
   788			return nil, err
   789		}
   790		hreq := c.newRequest(ctx, "POST", url, bytes.NewReader(body))
   791		return c.expect2XX(hreq)
   792	}
   793	
   794	func (c *Client) Query(ctx context.Context, req *search.SearchQuery) (*search.SearchResult, error) {
   795		hres, err := c.query(ctx, req)
   796		if err != nil {
   797			return nil, err
   798		}
   799		res := new(search.SearchResult)
   800		if err := httputil.DecodeJSON(hres, res); err != nil {
   801			return nil, err
   802		}
   803		return res, nil
   804	}
   805	
   806	// QueryRaw sends req and returns the body of the response, which should be the
   807	// unparsed JSON of a search.SearchResult.
   808	func (c *Client) QueryRaw(ctx context.Context, req *search.SearchQuery) ([]byte, error) {
   809		hres, err := c.query(ctx, req)
   810		if err != nil {
   811			return nil, err
   812		}
   813		defer hres.Body.Close()
   814		return ioutil.ReadAll(hres.Body)
   815	}
   816	
   817	// SearchExistingFileSchema does a search query looking for an
   818	// existing file with entire contents of wholeRef, then does a HEAD
   819	// request to verify the file still exists on the server. If so,
   820	// it returns that file schema's blobref.
   821	//
   822	// If multiple wholeRef values are provided, any may match. This is
   823	// used for searching for the file by multiple wholeRef hashes
   824	// (e.g. SHA-224 and SHA-1).
   825	//
   826	// It returns (zero, nil) if not found. A non-nil error is only returned
   827	// if there were problems searching.
   828	func (c *Client) SearchExistingFileSchema(ctx context.Context, wholeRef ...blob.Ref) (blob.Ref, error) {
   829		sr, err := c.SearchRoot()
   830		if err != nil {
   831			return blob.Ref{}, err
   832		}
   833		if len(wholeRef) == 0 {
   834			return blob.Ref{}, nil
   835		}
   836		url := sr + "camli/search/files"
   837		for i, ref := range wholeRef {
   838			if i == 0 {
   839				url += "?wholedigest=" + ref.String()
   840			} else {
   841				url += "&wholedigest=" + ref.String()
   842			}
   843		}
   844		req := c.newRequest(ctx, "GET", url)
   845		res, err := c.doReqGated(req)
   846		if err != nil {
   847			return blob.Ref{}, err
   848		}
   849		if res.StatusCode != 200 {
   850			body, _ := ioutil.ReadAll(io.LimitReader(res.Body, 1<<20))
   851			res.Body.Close()
   852			return blob.Ref{}, fmt.Errorf("client: got status code %d from URL %s; body %s", res.StatusCode, url, body)
   853		}
   854		var ress camtypes.FileSearchResponse
   855		if err := httputil.DecodeJSON(res, &ress); err != nil {
   856			// Check that we're not just hitting the change introduced in 2018-01-13-6e8a5930c9fee81640c6c75a9a549fec98064186
   857			mismatch, err := c.versionMismatch(ctx)
   858			if err != nil {
   859				log.Printf("Could not verify whether client is too recent or server is too old: %v", err)
   860			} else if mismatch {
   861				return blob.Ref{}, fmt.Errorf("Client is too recent for this server. Use a client built before 2018-01-13-6e8a5930c9, or upgrade the server to after that revision.")
   862			}
   863			return blob.Ref{}, fmt.Errorf("client: error parsing JSON from URL %s: %v", url, err)
   864		}
   865		if len(ress.Files) == 0 {
   866			return blob.Ref{}, nil
   867		}
   868		for wholeRef, files := range ress.Files {
   869			for _, f := range files {
   870				if c.FileHasContents(ctx, f, blob.MustParse(wholeRef)) {
   871					return f, nil
   872				}
   873			}
   874		}
   875		return blob.Ref{}, nil
   876	}
   877	
   878	// versionMismatch returns true if the server was built before 2018-01-13 and
   879	// the client was built at or after 2018-01-13.
   880	func (c *Client) versionMismatch(ctx context.Context) (bool, error) {
   881		const shortRFC3339 = "2006-01-02"
   882		version := buildinfo.GitInfo
   883		if version == "" {
   884			return false, errors.New("unknown client version")
   885		}
   886		version = version[:10] // keep only the date part
   887		clientDate, err := time.Parse(shortRFC3339, version)
   888		if err != nil {
   889			return false, fmt.Errorf("could not parse date from version %q: %v", version, err)
   890		}
   891		apiChangeDate, _ := time.Parse(shortRFC3339, "2018-01-13")
   892		if !clientDate.After(apiChangeDate) {
   893			// client is old enough, all good.
   894			return false, nil
   895		}
   896		url := c.discoRoot() + "/status/status.json"
   897		req := c.newRequest(ctx, "GET", url)
   898		res, err := c.doReqGated(req)
   899		if err != nil {
   900			return false, err
   901		}
   902		if res.StatusCode != 200 {
   903			body, _ := ioutil.ReadAll(io.LimitReader(res.Body, 1<<20))
   904			res.Body.Close()
   905			return false, fmt.Errorf("got status code %d from URL %s; body %s", res.StatusCode, url, body)
   906		}
   907		var status struct {
   908			Version string `json:"version"`
   909		}
   910		if err := httputil.DecodeJSON(res, &status); err != nil {
   911			return false, fmt.Errorf("error parsing JSON from URL %s: %v", url, err)
   912		}
   913		serverVersion := status.Version[:10]
   914		serverDate, err := time.Parse(shortRFC3339, serverVersion)
   915		if err != nil {
   916			return false, fmt.Errorf("could not parse date from server version %q: %v", status.Version, err)
   917		}
   918		if serverDate.After(apiChangeDate) {
   919			// server is recent enough, all good.
   920			return false, nil
   921		}
   922		return true, nil
   923	}
   924	
   925	// FileHasContents returns true iff f refers to a "file" or "bytes" schema blob,
   926	// the server is configured with a "download helper", and the server responds
   927	// that all chunks of 'f' are available and match the digest of wholeRef.
   928	func (c *Client) FileHasContents(ctx context.Context, f, wholeRef blob.Ref) bool {
   929		if err := c.condDiscovery(); err != nil {
   930			return false
   931		}
   932		if c.downloadHelper == "" {
   933			return false
   934		}
   935		req := c.newRequest(ctx, "HEAD", c.downloadHelper+f.String()+"/?verifycontents="+wholeRef.String())
   936		res, err := c.expect2XX(req)
   937		if err != nil {
   938			log.Printf("download helper HEAD error: %v", err)
   939			return false
   940		}
   941		defer res.Body.Close()
   942		return res.Header.Get("X-Camli-Contents") == wholeRef.String()
   943	}
   944	
   945	// prefix returns the URL prefix before "/camli/", or before
   946	// the blobref hash in case of a share URL.
   947	// Examples: http://foo.com:3179/bs or http://foo.com:3179/share
   948	func (c *Client) prefix() (string, error) {
   949		if err := c.prefixOnce.Do(c.initPrefix); err != nil {
   950			return "", err
   951		}
   952		return c.prefixv, nil
   953	}
   954	
   955	// blobPrefix returns the URL prefix before the blobref hash.
   956	// Example: http://foo.com:3179/bs/camli or http://foo.com:3179/share
   957	func (c *Client) blobPrefix() (string, error) {
   958		pfx, err := c.prefix()
   959		if err != nil {
   960			return "", err
   961		}
   962		if !c.isSharePrefix {
   963			pfx += "/camli"
   964		}
   965		return pfx, nil
   966	}
   967	
   968	// discoRoot returns the user defined server for this client. It prepends "https://" if no scheme was specified.
   969	func (c *Client) discoRoot() string {
   970		s := c.server
   971		if c.sameOrigin {
   972			s = strings.TrimPrefix(s, "http://")
   973			s = strings.TrimPrefix(s, "https://")
   974			parts := strings.SplitN(s, "/", 1)
   975			if len(parts) < 2 {
   976				return "/"
   977			}
   978			return "/" + parts[1]
   979		}
   980		if !strings.HasPrefix(s, "http") {
   981			s = "https://" + s
   982		}
   983		return s
   984	}
   985	
   986	// initPrefix uses the user provided server URL to define the URL
   987	// prefix to the blobserver root. If the server URL has a path
   988	// component then it is directly used, otherwise the blobRoot
   989	// from the discovery is used as the path.
   990	func (c *Client) initPrefix() error {
   991		c.isSharePrefix = false
   992		root := c.discoRoot()
   993		u, err := url.Parse(root)
   994		if err != nil {
   995			return err
   996		}
   997		if len(u.Path) > 1 {
   998			c.prefixv = strings.TrimRight(root, "/")
   999			return nil
  1000		}
  1001		return c.condDiscovery()
  1002	}
  1003	
  1004	func (c *Client) condDiscovery() error {
  1005		if c.sto != nil {
  1006			return errors.New("client not using HTTP")
  1007		}
  1008		return c.discoOnce.Do(c.doDiscovery)
  1009	}
  1010	
  1011	// DiscoveryDoc returns the server's JSON discovery document.
  1012	// This method exists purely for the "camtool discovery" command.
  1013	// Clients shouldn't have to parse this themselves.
  1014	func (c *Client) DiscoveryDoc(ctx context.Context) (io.Reader, error) {
  1015		res, err := c.discoveryResp(ctx)
  1016		if err != nil {
  1017			return nil, err
  1018		}
  1019		defer res.Body.Close()
  1020		const maxSize = 1 << 20
  1021		all, err := ioutil.ReadAll(io.LimitReader(res.Body, maxSize+1))
  1022		if err != nil {
  1023			return nil, err
  1024		}
  1025		if len(all) > maxSize {
  1026			return nil, errors.New("discovery document oddly large")
  1027		}
  1028		if len(all) > 0 && all[len(all)-1] != '\n' {
  1029			all = append(all, '\n')
  1030		}
  1031		return bytes.NewReader(all), err
  1032	}
  1033	
  1034	// HTTPVersion reports the HTTP version in use, such as "HTTP/1.1" or "HTTP/2.0".
  1035	func (c *Client) HTTPVersion(ctx context.Context) (string, error) {
  1036		req := c.newRequest(ctx, "HEAD", c.discoRoot(), nil)
  1037		res, err := c.doReqGated(req)
  1038		if err != nil {
  1039			return "", err
  1040		}
  1041		return res.Proto, err
  1042	}
  1043	
  1044	func (c *Client) discoveryResp(ctx context.Context) (*http.Response, error) {
  1045		// If the path is just "" or "/", do discovery against
  1046		// the URL to see which path we should actually use.
  1047		req := c.newRequest(ctx, "GET", c.discoRoot(), nil)
  1048		req.Header.Set("Accept", "text/x-camli-configuration")
  1049		res, err := c.doReqGated(req)
  1050		if err != nil {
  1051			return nil, err
  1052		}
  1053		if res.StatusCode != 200 {
  1054			res.Body.Close()
  1055			errMsg := fmt.Sprintf("got status %q from blobserver URL %q during configuration discovery", res.Status, c.discoRoot())
  1056			if res.StatusCode == 401 && c.authErr != nil {
  1057				errMsg = fmt.Sprintf("%v. %v", c.authErr, errMsg)
  1058			}
  1059			return nil, errors.New(errMsg)
  1060		}
  1061		// TODO(bradfitz): little weird in retrospect that we request
  1062		// text/x-camli-configuration and expect to get back
  1063		// text/javascript.  Make them consistent.
  1064		if ct := res.Header.Get("Content-Type"); ct != "text/javascript" {
  1065			res.Body.Close()
  1066			return nil, fmt.Errorf("Blobserver returned unexpected type %q from discovery", ct)
  1067		}
  1068		return res, nil
  1069	}
  1070	
  1071	func (c *Client) doDiscovery() error {
  1072		ctx := context.TODO()
  1073		root, err := url.Parse(c.discoRoot())
  1074		if err != nil {
  1075			return err
  1076		}
  1077	
  1078		res, err := c.discoveryResp(ctx)
  1079		if err != nil {
  1080			return err
  1081		}
  1082	
  1083		var disco camtypes.Discovery
  1084		if err := httputil.DecodeJSON(res, &disco); err != nil {
  1085			return err
  1086		}
  1087	
  1088		if disco.SearchRoot == "" {
  1089			c.searchRoot = ""
  1090		} else {
  1091			u, err := root.Parse(disco.SearchRoot)
  1092			if err != nil {
  1093				return fmt.Errorf("client: invalid searchRoot %q; failed to resolve", disco.SearchRoot)
  1094			}
  1095			c.searchRoot = u.String()
  1096		}
  1097	
  1098		u, err := root.Parse(disco.HelpRoot)
  1099		if err != nil {
  1100			return fmt.Errorf("client: invalid helpRoot %q; failed to resolve", disco.HelpRoot)
  1101		}
  1102		c.helpRoot = u.String()
  1103	
  1104		u, err = root.Parse(disco.ShareRoot)
  1105		if err != nil {
  1106			return fmt.Errorf("client: invalid shareRoot %q; failed to resolve", disco.ShareRoot)
  1107		}
  1108		c.shareRoot = u.String()
  1109	
  1110		c.storageGen = disco.StorageGeneration
  1111		c.hasLegacySHA1 = disco.HasLegacySHA1Index
  1112	
  1113		u, err = root.Parse(disco.BlobRoot)
  1114		if err != nil {
  1115			return fmt.Errorf("client: error resolving blobRoot: %v", err)
  1116		}
  1117		c.prefixv = strings.TrimRight(u.String(), "/")
  1118	
  1119		if disco.UIDiscovery != nil {
  1120			u, err = root.Parse(disco.DownloadHelper)
  1121			if err != nil {
  1122				return fmt.Errorf("client: invalid downloadHelper %q; failed to resolve", disco.DownloadHelper)
  1123			}
  1124			c.downloadHelper = u.String()
  1125		}
  1126	
  1127		if disco.SyncHandlers != nil {
  1128			for _, v := range disco.SyncHandlers {
  1129				ufrom, err := root.Parse(v.From)
  1130				if err != nil {
  1131					return fmt.Errorf("client: invalid %q \"from\" sync; failed to resolve", v.From)
  1132				}
  1133				uto, err := root.Parse(v.To)
  1134				if err != nil {
  1135					return fmt.Errorf("client: invalid %q \"to\" sync; failed to resolve", v.To)
  1136				}
  1137				c.syncHandlers = append(c.syncHandlers, &SyncInfo{
  1138					From:    ufrom.String(),
  1139					To:      uto.String(),
  1140					ToIndex: v.ToIndex,
  1141				})
  1142			}
  1143		}
  1144	
  1145		if disco.Signing != nil {
  1146			c.serverKeyID = disco.Signing.PublicKeyID
  1147			c.serverPublicKeyBlobRef = disco.Signing.PublicKeyBlobRef
  1148			c.signHandler = disco.Signing.SignHandler
  1149		}
  1150		return nil
  1151	}
  1152	
  1153	// GetJSON sends a GET request to url, and unmarshals the returned
  1154	// JSON response into data. The URL's host must match the client's
  1155	// configured server.
  1156	func (c *Client) GetJSON(ctx context.Context, url string, data interface{}) error {
  1157		if !strings.HasPrefix(url, c.discoRoot()) {
  1158			return fmt.Errorf("wrong URL (%q) for this server", url)
  1159		}
  1160		hreq := c.newRequest(ctx, "GET", url)
  1161		resp, err := c.expect2XX(hreq)
  1162		if err != nil {
  1163			return err
  1164		}
  1165		return httputil.DecodeJSON(resp, data)
  1166	}
  1167	
  1168	// Post is like http://golang.org/pkg/net/http/#Client.Post
  1169	// but with implementation details like gated requests. The
  1170	// URL's host must match the client's configured server.
  1171	func (c *Client) Post(ctx context.Context, url string, bodyType string, body io.Reader) error {
  1172		resp, err := c.post(ctx, url, bodyType, body)
  1173		if err != nil {
  1174			return err
  1175		}
  1176		return resp.Body.Close()
  1177	}
  1178	
  1179	// Sign sends a request to the sign handler on server to sign the contents of r,
  1180	// and return them signed. It uses the same implementation details, such as gated
  1181	// requests, as Post.
  1182	func (c *Client) Sign(ctx context.Context, server string, r io.Reader) (signed []byte, err error) {
  1183		signHandler, err := c.SignHandler()
  1184		if err != nil {
  1185			return nil, err
  1186		}
  1187		signServer := strings.TrimSuffix(server, "/") + signHandler
  1188		resp, err := c.post(ctx, signServer, "application/x-www-form-urlencoded", r)
  1189		if err != nil {
  1190			return nil, err
  1191		}
  1192		defer resp.Body.Close()
  1193		return ioutil.ReadAll(resp.Body)
  1194	}
  1195	
  1196	func (c *Client) post(ctx context.Context, url string, bodyType string, body io.Reader) (*http.Response, error) {
  1197		if !c.sameOrigin && !strings.HasPrefix(url, c.discoRoot()) {
  1198			return nil, fmt.Errorf("wrong URL (%q) for this server", url)
  1199		}
  1200		req := c.newRequest(ctx, "POST", url, body)
  1201		req.Header.Set("Content-Type", bodyType)
  1202		res, err := c.expect2XX(req)
  1203		if err != nil {
  1204			return nil, err
  1205		}
  1206		return res, nil
  1207	}
  1208	
  1209	// newRequest creates a request with the authentication header, and with the
  1210	// appropriate scheme and port in the case of self-signed TLS.
  1211	func (c *Client) newRequest(ctx context.Context, method, url string, body ...io.Reader) *http.Request {
  1212		var bodyR io.Reader
  1213		if len(body) > 0 {
  1214			bodyR = body[0]
  1215		}
  1216		if len(body) > 1 {
  1217			panic("too many body arguments")
  1218		}
  1219		req, err := http.NewRequest(method, url, bodyR)
  1220		if err != nil {
  1221			panic(err.Error())
  1222		}
  1223		// not done by http.NewRequest in Go 1.0:
  1224		if br, ok := bodyR.(*bytes.Reader); ok {
  1225			req.ContentLength = int64(br.Len())
  1226		}
  1227		c.authMode.AddAuthHeader(req)
  1228		return req.WithContext(ctx)
  1229	}
  1230	
  1231	// expect2XX will doReqGated and promote HTTP response codes outside of
  1232	// the 200-299 range to a non-nil error containing the response body.
  1233	func (c *Client) expect2XX(req *http.Request) (*http.Response, error) {
  1234		res, err := c.doReqGated(req)
  1235		if err == nil && (res.StatusCode < 200 || res.StatusCode > 299) {
  1236			buf := new(bytes.Buffer)
  1237			io.CopyN(buf, res.Body, 1<<20)
  1238			res.Body.Close()
  1239			return res, fmt.Errorf("client: got status code %d from URL %s; body %s", res.StatusCode, req.URL.String(), buf.String())
  1240		}
  1241		return res, err
  1242	}
  1243	
  1244	func (c *Client) doReqGated(req *http.Request) (*http.Response, error) {
  1245		c.httpGate.Start()
  1246		defer c.httpGate.Done()
  1247		return c.httpClient.Do(req)
  1248	}
  1249	
  1250	// DialFunc returns the adequate dial function when we're on android.
  1251	func (c *Client) DialFunc() func(network, addr string) (net.Conn, error) {
  1252		if c.useTLS() {
  1253			return nil
  1254		}
  1255		if android.IsChild() {
  1256			return func(network, addr string) (net.Conn, error) {
  1257				return android.Dial(network, addr)
  1258			}
  1259		}
  1260		return nil
  1261	}
  1262	
  1263	func (c *Client) http2DialTLSFunc() func(network, addr string, cfg *tls.Config) (net.Conn, error) {
  1264		trustedCerts := c.getTrustedCerts()
  1265		if !c.insecureAnyTLSCert && len(trustedCerts) == 0 {
  1266			// TLS with normal/full verification.
  1267			// nil means http2 uses its default dialer.
  1268			return nil
  1269		}
  1270		return func(network, addr string, cfg *tls.Config) (net.Conn, error) {
  1271			// we own cfg, so we can mutate it:
  1272			cfg.InsecureSkipVerify = true
  1273			conn, err := tls.Dial(network, addr, cfg)
  1274			if err != nil {
  1275				return nil, err
  1276			}
  1277			if c.insecureAnyTLSCert {
  1278				return conn, err
  1279			}
  1280			state := conn.ConnectionState()
  1281			if p := state.NegotiatedProtocol; p != http2.NextProtoTLS {
  1282				return nil, fmt.Errorf("http2: unexpected ALPN protocol %q; want %q", p, http2.NextProtoTLS)
  1283			}
  1284			if !state.NegotiatedProtocolIsMutual {
  1285				return nil, errors.New("http2: could not negotiate protocol mutually")
  1286			}
  1287			certs := state.PeerCertificates
  1288			if len(certs) < 1 {
  1289				return nil, fmt.Errorf("no TLS peer certificates from %s", addr)
  1290			}
  1291			sig := hashutil.SHA256Prefix(certs[0].Raw)
  1292			for _, v := range trustedCerts {
  1293				if v == sig {
  1294					return conn, nil
  1295				}
  1296			}
  1297			return nil, fmt.Errorf("TLS server at %v presented untrusted certificate (signature %q)", addr, sig)
  1298		}
  1299	}
  1300	
  1301	// DialTLSFunc returns the adequate dial function, when using SSL, depending on
  1302	// whether we're using insecure TLS (certificate verification is disabled), or we
  1303	// have some trusted certs, or we're on android.
  1304	// If the client's config has some trusted certs, the server's certificate will
  1305	// be checked against those in the config after the TLS handshake.
  1306	func (c *Client) DialTLSFunc() func(network, addr string) (net.Conn, error) {
  1307		if !c.useTLS() {
  1308			return nil
  1309		}
  1310		trustedCerts := c.getTrustedCerts()
  1311		var stdTLS bool
  1312		if !c.insecureAnyTLSCert && len(trustedCerts) == 0 {
  1313			// TLS with normal/full verification.
  1314			stdTLS = true
  1315			if !android.IsChild() {
  1316				// Not android, so let the stdlib deal with it
  1317				return nil
  1318			}
  1319		}
  1320	
  1321		return func(network, addr string) (net.Conn, error) {
  1322			var conn *tls.Conn
  1323			var err error
  1324			if android.IsChild() {
  1325				ac, err := android.Dial(network, addr)
  1326				if err != nil {
  1327					return nil, err
  1328				}
  1329				var tlsConfig *tls.Config
  1330				if stdTLS {
  1331					tlsConfig, err = android.TLSConfig()
  1332					if err != nil {
  1333						return nil, err
  1334					}
  1335				} else {
  1336					tlsConfig = &tls.Config{InsecureSkipVerify: true}
  1337				}
  1338				// Since we're doing the TLS handshake ourselves, we need to set the ServerName,
  1339				// in case the server uses SNI (as is the case if it's relying on Let's Encrypt,
  1340				// for example).
  1341				tlsConfig.ServerName = c.serverNameOfAddr(addr)
  1342				conn = tls.Client(ac, tlsConfig)
  1343				if err := conn.Handshake(); err != nil {
  1344					return nil, err
  1345				}
  1346				if stdTLS {
  1347					// Normal TLS verification succeeded and we do not have
  1348					// additional trusted certificate fingerprints to check for.
  1349					return conn, nil
  1350				}
  1351			} else {
  1352				conn, err = tls.Dial(network, addr, &tls.Config{InsecureSkipVerify: true})
  1353				if err != nil {
  1354					return nil, err
  1355				}
  1356			}
  1357			if c.insecureAnyTLSCert {
  1358				return conn, nil
  1359			}
  1360			certs := conn.ConnectionState().PeerCertificates
  1361			if len(certs) < 1 {
  1362				return nil, fmt.Errorf("no TLS peer certificates from %s", addr)
  1363			}
  1364			sig := hashutil.SHA256Prefix(certs[0].Raw)
  1365			for _, v := range trustedCerts {
  1366				if v == sig {
  1367					return conn, nil
  1368				}
  1369			}
  1370			return nil, fmt.Errorf("TLS server at %v presented untrusted certificate (signature %q)", addr, sig)
  1371		}
  1372	}
  1373	
  1374	// serverNameOfAddr returns the host part of addr, or the empty string if addr
  1375	// is not a valid address (see net.Dial). Additionally, if host is an IP literal,
  1376	// serverNameOfAddr returns the empty string.
  1377	func (c *Client) serverNameOfAddr(addr string) string {
  1378		serverName, _, err := net.SplitHostPort(addr)
  1379		if err != nil {
  1380			c.printf("could not get server name from address %q: %v", addr, err)
  1381			return ""
  1382		}
  1383		if ip := net.ParseIP(serverName); ip != nil {
  1384			return ""
  1385		}
  1386		return serverName
  1387	}
  1388	
  1389	// Signer returns the client's Signer, if any. The Signer signs JSON
  1390	// mutation claims.
  1391	func (c *Client) Signer() (*schema.Signer, error) {
  1392		c.signerOnce.Do(c.signerInit)
  1393		return c.signer, c.signerErr
  1394	}
  1395	
  1396	func (c *Client) signerInit() {
  1397		c.signer, c.signerErr = c.buildSigner()
  1398	}
  1399	
  1400	func (c *Client) buildSigner() (*schema.Signer, error) {
  1401		c.initSignerPublicKeyBlobrefOnce.Do(c.initSignerPublicKeyBlobref)
  1402		if !c.signerPublicKeyRef.Valid() {
  1403			return nil, camtypes.ErrClientNoPublicKey
  1404		}
  1405		return schema.NewSigner(c.signerPublicKeyRef, strings.NewReader(c.publicKeyArmored), c.SecretRingFile())
  1406	}
  1407	
  1408	// sigTime optionally specifies the signature time.
  1409	// If zero, the current time is used.
  1410	func (c *Client) signBlob(ctx context.Context, bb schema.Buildable, sigTime time.Time) (string, error) {
  1411		signer, err := c.Signer()
  1412		if err != nil {
  1413			return "", err
  1414		}
  1415		return bb.Builder().SignAt(ctx, signer, sigTime)
  1416	}
  1417	
  1418	// uploadPublicKey uploads the public key (if one is defined), so
  1419	// subsequent (likely synchronous) indexing of uploaded signed blobs
  1420	// will have access to the public key to verify it. In the normal
  1421	// case, the stat cache prevents this from doing anything anyway.
  1422	func (c *Client) uploadPublicKey(ctx context.Context) error {
  1423		sigRef := c.SignerPublicKeyBlobref()
  1424		if !sigRef.Valid() {
  1425			return nil
  1426		}
  1427		var err error
  1428		if _, keyUploaded := c.haveCache.StatBlobCache(sigRef); !keyUploaded {
  1429			_, err = c.uploadString(ctx, c.publicKeyArmored, false)
  1430		}
  1431		return err
  1432	}
  1433	
  1434	// checkMatchingKeys compares the client's and the server's keys and logs if they differ.
  1435	func (c *Client) checkMatchingKeys() {
  1436		serverKey, err := c.ServerKeyID()
  1437		if err != nil {
  1438			log.Printf("Warning: Could not obtain the server's key id: %v", err)
  1439			return
  1440		}
  1441		if serverKey != c.signer.KeyIDLong() {
  1442			log.Printf("Warning: client (%s) and server (%s) keys differ.", c.signer.KeyIDLong(), serverKey)
  1443		}
  1444	}
  1445	
  1446	func (c *Client) UploadAndSignBlob(ctx context.Context, b schema.AnyBlob) (*PutResult, error) {
  1447		signed, err := c.signBlob(ctx, b.Blob(), time.Time{})
  1448		if err != nil {
  1449			return nil, err
  1450		}
  1451		c.checkMatchingKeys()
  1452		if err := c.uploadPublicKey(ctx); err != nil {
  1453			return nil, err
  1454		}
  1455		return c.uploadString(ctx, signed, false)
  1456	}
  1457	
  1458	func (c *Client) UploadBlob(ctx context.Context, b schema.AnyBlob) (*PutResult, error) {
  1459		// TODO(bradfitz): ask the blob for its own blobref, rather
  1460		// than changing the hash function with uploadString?
  1461		return c.uploadString(ctx, b.Blob().JSON(), true)
  1462	}
  1463	
  1464	func (c *Client) uploadString(ctx context.Context, s string, stat bool) (*PutResult, error) {
  1465		uh := NewUploadHandleFromString(s)
  1466		uh.SkipStat = !stat
  1467		return c.Upload(ctx, uh)
  1468	}
  1469	
  1470	func (c *Client) UploadNewPermanode(ctx context.Context) (*PutResult, error) {
  1471		unsigned := schema.NewUnsignedPermanode()
  1472		return c.UploadAndSignBlob(ctx, unsigned)
  1473	}
  1474	
  1475	func (c *Client) UploadPlannedPermanode(ctx context.Context, key string, sigTime time.Time) (*PutResult, error) {
  1476		unsigned := schema.NewPlannedPermanode(key)
  1477		signed, err := c.signBlob(ctx, unsigned, sigTime)
  1478		if err != nil {
  1479			return nil, err
  1480		}
  1481		c.checkMatchingKeys()
  1482		if err := c.uploadPublicKey(ctx); err != nil {
  1483			return nil, err
  1484		}
  1485		return c.uploadString(ctx, signed, true)
  1486	}
  1487	
  1488	// IsIgnoredFile returns whether the file at fullpath should be ignored by pk-put.
  1489	// The fullpath is checked against the ignoredFiles list, trying the following rules in this order:
  1490	// 1) star-suffix style matching (.e.g *.jpg).
  1491	// 2) Shell pattern match as done by http://golang.org/pkg/path/filepath/#Match
  1492	// 3) If the pattern is an absolute path to a directory, fullpath matches if it is that directory or a child of it.
  1493	// 4) If the pattern is a relative path, fullpath matches if it has pattern as a path component (i.e the pattern is a part of fullpath that fits exactly between two path separators).
  1494	func (c *Client) IsIgnoredFile(fullpath string) bool {
  1495		c.initIgnoredFilesOnce.Do(c.initIgnoredFiles)
  1496		return c.ignoreChecker(fullpath)
  1497	}
  1498	
  1499	// Close closes the client.
  1500	func (c *Client) Close() error {
  1501		if cl, ok := c.sto.(io.Closer); ok {
  1502			return cl.Close()
  1503		}
  1504		if c := c.HTTPClient(); c != nil {
  1505			switch t := c.Transport.(type) {
  1506			case *http.Transport:
  1507				t.CloseIdleConnections()
  1508			case *http2.Transport:
  1509				t.CloseIdleConnections()
  1510			}
  1511		}
  1512		return nil
  1513	}
  1514	
  1515	func httpGateSize(rt http.RoundTripper) int {
  1516		switch v := rt.(type) {
  1517		case *httputil.StatsTransport:
  1518			return httpGateSize(v.Transport)
  1519		case *http.Transport:
  1520			return maxParallelHTTP_h1
  1521		case *http2.Transport:
  1522			return maxParallelHTTP_h2
  1523		default:
  1524			return maxParallelHTTP_h1 // conservative default
  1525		}
  1526	}