Home Download Docs Code Community
     1	/*
     2	Copyright 2011 The Perkeep Authors.
     3	
     4	Licensed under the Apache License, Version 2.0 (the "License");
     5	you may not use this file except in compliance with the License.
     6	You may obtain a copy of the License at
     7	
     8	     http://www.apache.org/licenses/LICENSE-2.0
     9	
    10	Unless required by applicable law or agreed to in writing, software
    11	distributed under the License is distributed on an "AS IS" BASIS,
    12	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13	See the License for the specific language governing permissions and
    14	limitations under the License.
    15	*/
    16	
    17	// Package client implements a Perkeep client.
    18	package client // import "perkeep.org/pkg/client"
    19	
    20	import (
    21		"bytes"
    22		"context"
    23		"crypto/tls"
    24		"encoding/json"
    25		"errors"
    26		"fmt"
    27		"io"
    28		"io/ioutil"
    29		"log"
    30		"net"
    31		"net/http"
    32		"net/url"
    33		"os"
    34		"regexp"
    35		"strings"
    36		"sync"
    37		"time"
    38	
    39		"perkeep.org/internal/hashutil"
    40		"perkeep.org/internal/httputil"
    41		"perkeep.org/internal/osutil"
    42		"perkeep.org/pkg/auth"
    43		"perkeep.org/pkg/blob"
    44		"perkeep.org/pkg/blobserver"
    45		"perkeep.org/pkg/buildinfo"
    46		"perkeep.org/pkg/client/android"
    47		"perkeep.org/pkg/schema"
    48		"perkeep.org/pkg/search"
    49		"perkeep.org/pkg/types/camtypes"
    50	
    51		"go4.org/syncutil"
    52		"golang.org/x/net/http2"
    53	)
    54	
    55	// A Client provides access to a Perkeep server.
    56	//
    57	// After use, a Client should be closed via its Close method to
    58	// release idle HTTP connections or other resourcedds.
    59	type Client struct {
    60		// server is the input from user, pre-discovery.
    61		// For example "http://foo.com" or "foo.com:1234".
    62		// It is the responsibility of initPrefix to parse
    63		// server and set prefix, including doing discovery
    64		// to figure out what the proper server-declared
    65		// prefix is.
    66		server string
    67	
    68		prefixOnce    syncutil.Once // guards init of following 2 fields
    69		prefixv       string        // URL prefix before "/camli/"
    70		isSharePrefix bool          // URL is a request for a share blob
    71	
    72		discoOnce              syncutil.Once
    73		searchRoot             string      // Handler prefix, or "" if none
    74		downloadHelper         string      // or "" if none
    75		storageGen             string      // storage generation, or "" if not reported
    76		hasLegacySHA1          bool        // Whether server has SHA-1 blobs indexed.
    77		syncHandlers           []*SyncInfo // "from" and "to" url prefix for each syncHandler
    78		serverKeyID            string      // Server's GPG public key ID.
    79		helpRoot               string      // Handler prefix, or "" if none
    80		shareRoot              string      // Share handler prefix, or "" if none
    81		serverPublicKeyBlobRef blob.Ref    // Server's public key blobRef
    82	
    83		signerOnce  sync.Once
    84		signer      *schema.Signer
    85		signerErr   error
    86		signHandler string // Handler prefix, or "" if none
    87	
    88		authMode auth.AuthMode
    89		// authErr is set when no auth config is found but we want to defer warning
    90		// until discovery fails.
    91		authErr error
    92	
    93		httpClient *http.Client
    94		haveCache  HaveCache
    95	
    96		// If sto is set, it's used before the httpClient or other network operations.
    97		sto blobserver.Storage
    98	
    99		initTrustedCertsOnce sync.Once
   100	
   101		// We define a certificate fingerprint as the 20 digits lowercase prefix
   102		// of the SHA256 of the complete certificate (in ASN.1 DER encoding).
   103		// trustedCerts contains the fingerprints of the self-signed
   104		// certificates we trust.
   105		// If not empty, (and if using TLS) the full x509 verification is
   106		// disabled, and we instead check the server's certificate against
   107		// this list.
   108		// The perkeepd server prints the fingerprint to add to the config
   109		// when starting.
   110		trustedCerts []string
   111	
   112		// insecureAnyTLSCert disables all TLS cert checking,
   113		// including the trustedCerts field above.
   114		insecureAnyTLSCert bool
   115	
   116		initIgnoredFilesOnce sync.Once
   117		// list of files that pk-put should ignore.
   118		// Defaults to empty, but pk-put init creates a config with a non
   119		// empty list.
   120		// See IsIgnoredFile for the matching rules.
   121		ignoredFiles  []string
   122		ignoreChecker func(path string) bool
   123	
   124		pendStatMu sync.Mutex             // guards pendStat
   125		pendStat   map[blob.Ref][]statReq // blobref -> reqs; for next batch(es)
   126	
   127		initSignerPublicKeyBlobrefOnce sync.Once
   128		signerPublicKeyRef             blob.Ref
   129		publicKeyArmored               string
   130	
   131		statsMutex sync.Mutex
   132		stats      Stats
   133	
   134		// via maps the access path from a share root to a desired target.
   135		// It is non-nil when in "sharing" mode, where the Client is fetching
   136		// a share.
   137		viaMu sync.RWMutex
   138		via   map[blob.Ref]blob.Ref // target => via (target is referenced from via)
   139	
   140		// Verbose controls how much logging from the client is printed. The caller
   141		// should set it only before using the client, and treat it as read-only after
   142		// that.
   143		Verbose bool
   144		// Logger is the logger used by the client. It defaults to a standard
   145		// logger to os.Stderr if the client is initialized by one of the package's
   146		// functions. Like Verbose, it should be set only before using the client, and
   147		// not be modified afterwards.
   148		Logger *log.Logger
   149	
   150		httpGate        *syncutil.Gate
   151		transportConfig *TransportConfig // or nil
   152	
   153		noExtConfig bool // no external config; config file and env vars are ignored.
   154	
   155		// sameOrigin indicates whether URLs in requests should be stripped from
   156		// their scheme and HostPort parts. This is meant for when using the client
   157		// through gopherjs in the web UI. Because we'll run into CORS errors if
   158		// requests have a Host part.
   159		sameOrigin bool
   160	}
   161	
   162	const maxParallelHTTP_h1 = 5
   163	const maxParallelHTTP_h2 = 50
   164	
   165	// inGopherJS reports whether the client package is compiled by GopherJS, for use
   166	// in the browser.
   167	var inGopherJS bool
   168	
   169	// New returns a new Perkeep Client.
   170	//
   171	// By default, with no options, it uses the client as configured in
   172	// the environment or default configuration files.
   173	func New(opts ...ClientOption) (*Client, error) {
   174		c := &Client{
   175			haveCache: noHaveCache{},
   176			Logger:    log.New(os.Stderr, "", log.Ldate|log.Ltime),
   177			authMode:  auth.None{},
   178		}
   179		for _, v := range opts {
   180			v.modifyClient(c)
   181		}
   182		if c.sto != nil && len(opts) > 1 {
   183			return nil, errors.New("use of OptionUseStorageClient precludes use of any other options")
   184		}
   185	
   186		if inGopherJS {
   187			c.noExtConfig = true
   188			c.sameOrigin = true
   189		}
   190		if c.noExtConfig {
   191			c.setDefaultHTTPClient()
   192			return c, nil
   193		}
   194	
   195		if c.server != "" {
   196			if !isURLOrHostPort(c.server) {
   197				configOnce.Do(parseConfig)
   198				serverConf, ok := config.Servers[c.server]
   199				if !ok {
   200					log.Fatalf("%q looks like a server alias, but no such alias found in config at %v", c.server, osutil.UserClientConfigPath())
   201				}
   202				c.server = serverConf.Server
   203			}
   204			c.setDefaultHTTPClient()
   205			return c, nil
   206		}
   207	
   208		var err error
   209		c.server, err = getServer()
   210		if err != nil {
   211			return nil, err
   212		}
   213		err = c.SetupAuth()
   214		if err != nil {
   215			return nil, err
   216		}
   217		c.setDefaultHTTPClient()
   218		return c, nil
   219	}
   220	
   221	func (c *Client) setDefaultHTTPClient() {
   222		if c.httpClient == nil {
   223			c.httpClient = &http.Client{
   224				Transport: c.transportForConfig(c.transportConfig),
   225			}
   226		}
   227		c.httpGate = syncutil.NewGate(httpGateSize(c.httpClient.Transport))
   228	}
   229	
   230	// NewOrFail is like New, but calls log.Fatal instead of returning an error.
   231	func NewOrFail(opts ...ClientOption) *Client {
   232		c, err := New(opts...)
   233		if err != nil {
   234			log.Fatalf("error creating client: %v", err)
   235		}
   236		return c
   237	}
   238	
   239	// NewPathClient returns a new client accessing a subpath of c.
   240	func (c *Client) NewPathClient(path string) (*Client, error) {
   241		u, err := url.Parse(c.server)
   242		if err != nil {
   243			return nil, fmt.Errorf("bogus server %q for NewPathClient receiver: %v", c.server, err)
   244		}
   245		u.Path = path
   246		pc, err := New(OptionServer(u.String()))
   247		if err != nil {
   248			return nil, err
   249		}
   250		pc.authMode = c.authMode
   251		pc.discoOnce.Do(noop)
   252		return pc, nil
   253	}
   254	
   255	// TransportConfig contains options for how HTTP requests are made.
   256	type TransportConfig struct {
   257		// Proxy optionally specifies the Proxy for the transport. Useful with
   258		// pk-put for debugging even localhost requests.
   259		Proxy   func(*http.Request) (*url.URL, error)
   260		Verbose bool // Verbose enables verbose logging of HTTP requests.
   261	}
   262	
   263	func (c *Client) useHTTP2(tc *TransportConfig) bool {
   264		if !c.useTLS() {
   265			return false
   266		}
   267		if android.IsChild() {
   268			// No particular reason; just untested so far.
   269			return false
   270		}
   271		if os.Getenv("HTTPS_PROXY") != "" || os.Getenv("https_proxy") != "" ||
   272			(tc != nil && tc.Proxy != nil) {
   273			// Also just untested. Which proxies support h2 anyway?
   274			return false
   275		}
   276		return true
   277	}
   278	
   279	// transportForConfig returns a transport for the client, setting the correct
   280	// Proxy, Dial, and TLSClientConfig if needed. It does not mutate c.
   281	// It is the caller's responsibility to then use that transport to set
   282	// the client's httpClient with SetHTTPClient.
   283	func (c *Client) transportForConfig(tc *TransportConfig) http.RoundTripper {
   284		if inGopherJS {
   285			// Calls to net.Dial* functions - which would happen if the client's transport
   286			// is not nil - are prohibited with GopherJS. So we force nil here, so that the
   287			// call to transportForConfig in newClient is of no consequence when on the
   288			// browser.
   289			return nil
   290		}
   291		if c == nil {
   292			return nil
   293		}
   294		var transport http.RoundTripper
   295		proxy := http.ProxyFromEnvironment
   296		if tc != nil && tc.Proxy != nil {
   297			proxy = tc.Proxy
   298		}
   299	
   300		if c.useHTTP2(tc) {
   301			transport = &http2.Transport{
   302				DialTLS: c.http2DialTLSFunc(),
   303			}
   304		} else {
   305			transport = &http.Transport{
   306				DialTLS:             c.DialTLSFunc(),
   307				Dial:                c.DialFunc(),
   308				Proxy:               proxy,
   309				MaxIdleConnsPerHost: maxParallelHTTP_h1,
   310			}
   311		}
   312		httpStats := &httputil.StatsTransport{
   313			Transport: transport,
   314		}
   315		if tc != nil {
   316			httpStats.VerboseLog = tc.Verbose
   317		}
   318		transport = httpStats
   319		if android.IsChild() {
   320			transport = &android.StatsTransport{transport}
   321		}
   322		return transport
   323	}
   324	
   325	// HTTPStats returns the client's underlying httputil.StatsTransport, if in use.
   326	// If another transport is being used, nil is returned.
   327	func (c *Client) HTTPStats() *httputil.StatsTransport {
   328		st, _ := c.httpClient.Transport.(*httputil.StatsTransport)
   329		return st
   330	}
   331	
   332	type ClientOption interface {
   333		modifyClient(*Client)
   334	}
   335	
   336	// OptionServer returns a Client constructor option that forces use of
   337	// the provided server.
   338	//
   339	// The provided server is either "host:port" (assumed http, not https)
   340	// or a URL prefix, with or without a path, or a server alias from the
   341	// client configuration file. A server alias should not be confused
   342	// with a hostname, therefore it cannot contain any colon or period.
   343	func OptionServer(server string) ClientOption {
   344		return optionServer(server)
   345	}
   346	
   347	type optionServer string
   348	
   349	func (s optionServer) modifyClient(c *Client) { c.server = string(s) }
   350	
   351	// OptionUseStorageClient returns a Client constructor option that
   352	// forces use of the provided blob storage target.
   353	//
   354	// This exists mainly so all the convenience methods on
   355	// Client (e.g. the Upload variants) are available against storage
   356	// directly.
   357	//
   358	// Use of OptionUseStorageClient is mutually exclusively
   359	// with all other options, although it does imply
   360	// OptionNoExternalConfig(true).
   361	func OptionUseStorageClient(s blobserver.Storage) ClientOption {
   362		return optionStorage{s}
   363	}
   364	
   365	type optionStorage struct {
   366		sto blobserver.Storage
   367	}
   368	
   369	func (o optionStorage) modifyClient(c *Client) {
   370		c.sto = o.sto
   371		c.noExtConfig = true
   372	}
   373	
   374	// OptionTransportConfig returns a ClientOption that makes the client use
   375	// the provided transport configuration options.
   376	func OptionTransportConfig(tc *TransportConfig) ClientOption {
   377		return optionTransportConfig{tc}
   378	}
   379	
   380	type optionTransportConfig struct {
   381		tc *TransportConfig
   382	}
   383	
   384	func (o optionTransportConfig) modifyClient(c *Client) { c.transportConfig = o.tc }
   385	
   386	// OptionInsecure returns a ClientOption that controls whether HTTP
   387	// requests are allowed to be insecure (over HTTP or HTTPS without TLS
   388	// certificate checking). Use of this is strongly discouraged except
   389	// for local testing.
   390	func OptionInsecure(v bool) ClientOption {
   391		return optionInsecure(v)
   392	}
   393	
   394	type optionInsecure bool
   395	
   396	func (o optionInsecure) modifyClient(c *Client) { c.insecureAnyTLSCert = bool(o) }
   397	
   398	// OptionTrustedCert returns a ClientOption that makes the client
   399	// trust the provide self-signed cert signature. The value should be
   400	// the 20 byte hex prefix of the SHA-256 of the cert, as printed by
   401	// the perkeepd server on start-up.
   402	//
   403	// If cert is empty, the option has no effect.
   404	func OptionTrustedCert(cert string) ClientOption {
   405		// TODO: remove this whole function now that we have LetsEncrypt?
   406		return optionTrustedCert(cert)
   407	}
   408	
   409	type optionTrustedCert string
   410	
   411	func (o optionTrustedCert) modifyClient(c *Client) {
   412		cert := string(o)
   413		if cert != "" {
   414			c.initTrustedCertsOnce.Do(func() {})
   415			c.trustedCerts = []string{string(o)}
   416		}
   417	}
   418	
   419	type optionNoExtConfig bool
   420	
   421	func (o optionNoExtConfig) modifyClient(c *Client) { c.noExtConfig = bool(o) }
   422	
   423	// OptionNoExternalConfig returns a Client constructor option that
   424	// prevents any on-disk config files or environment variables from
   425	// influencing the client. It may still use the disk for caches.
   426	func OptionNoExternalConfig() ClientOption {
   427		return optionNoExtConfig(true)
   428	}
   429	
   430	type optionAuthMode struct {
   431		m auth.AuthMode
   432	}
   433	
   434	func (o optionAuthMode) modifyClient(c *Client) { c.authMode = o.m }
   435	
   436	// OptionAuthMode returns a Client constructor option that sets the
   437	// client's authentication mode.
   438	func OptionAuthMode(m auth.AuthMode) ClientOption {
   439		return optionAuthMode{m}
   440	}
   441	
   442	// noop is for use with syncutil.Onces.
   443	func noop() error { return nil }
   444	
   445	var shareURLRx = regexp.MustCompile(`^(.+)/(` + blob.Pattern + ")$")
   446	
   447	// NewFromShareRoot uses shareBlobURL to set up and return a client that
   448	// will be used to fetch shared blobs.
   449	func NewFromShareRoot(ctx context.Context, shareBlobURL string, opts ...ClientOption) (c *Client, target blob.Ref, err error) {
   450		var root string
   451		m := shareURLRx.FindStringSubmatch(shareBlobURL)
   452		if m == nil {
   453			return nil, blob.Ref{}, fmt.Errorf("Unknown share URL base")
   454		}
   455		c, err = New(append(opts[:len(opts):cap(opts)], OptionServer(m[1]))...)
   456		if err != nil {
   457			return nil, blob.Ref{}, err
   458		}
   459		c.discoOnce.Do(noop)
   460		c.prefixOnce.Do(noop)
   461		c.prefixv = m[1]
   462		c.isSharePrefix = true
   463		c.authMode = auth.None{}
   464		c.via = make(map[blob.Ref]blob.Ref)
   465		root = m[2]
   466	
   467		req := c.newRequest(ctx, "GET", shareBlobURL, nil)
   468		res, err := c.expect2XX(req)
   469		if err != nil {
   470			return nil, blob.Ref{}, fmt.Errorf("error fetching %s: %v", shareBlobURL, err)
   471		}
   472		defer res.Body.Close()
   473		var buf bytes.Buffer
   474		rootbr, ok := blob.Parse(root)
   475		if !ok {
   476			return nil, blob.Ref{}, fmt.Errorf("invalid root blob ref for sharing: %q", root)
   477		}
   478		b, err := schema.BlobFromReader(rootbr, io.TeeReader(res.Body, &buf))
   479		if err != nil {
   480			return nil, blob.Ref{}, fmt.Errorf("error parsing JSON from %s: %v , with response: %q", shareBlobURL, err, buf.Bytes())
   481		}
   482		if b.ShareAuthType() != schema.ShareHaveRef {
   483			return nil, blob.Ref{}, fmt.Errorf("unknown share authType of %q", b.ShareAuthType())
   484		}
   485		target = b.ShareTarget()
   486		if !target.Valid() {
   487			return nil, blob.Ref{}, fmt.Errorf("no target")
   488		}
   489		c.via[target] = rootbr
   490		return c, target, nil
   491	}
   492	
   493	// SetHTTPClient sets the Perkeep client's HTTP client.
   494	// If nil, the default HTTP client is used.
   495	func (c *Client) SetHTTPClient(client *http.Client) {
   496		if client == nil {
   497			client = http.DefaultClient
   498		}
   499		c.httpClient = client
   500	}
   501	
   502	// HTTPClient returns the Client's underlying http.Client.
   503	func (c *Client) HTTPClient() *http.Client {
   504		return c.httpClient
   505	}
   506	
   507	// A HaveCache caches whether a remote blobserver has a blob.
   508	type HaveCache interface {
   509		StatBlobCache(br blob.Ref) (size uint32, ok bool)
   510		NoteBlobExists(br blob.Ref, size uint32)
   511	}
   512	
   513	type noHaveCache struct{}
   514	
   515	func (noHaveCache) StatBlobCache(blob.Ref) (uint32, bool) { return 0, false }
   516	func (noHaveCache) NoteBlobExists(blob.Ref, uint32)       {}
   517	
   518	func (c *Client) SetHaveCache(cache HaveCache) {
   519		if cache == nil {
   520			cache = noHaveCache{}
   521		}
   522		c.haveCache = cache
   523	}
   524	
   525	func (c *Client) printf(format string, v ...interface{}) {
   526		if c.Verbose && c.Logger != nil {
   527			c.Logger.Printf(format, v...)
   528		}
   529	}
   530	
   531	func (c *Client) Stats() Stats {
   532		c.statsMutex.Lock()
   533		defer c.statsMutex.Unlock()
   534		return c.stats // copy
   535	}
   536	
   537	// ErrNoSearchRoot is returned by SearchRoot if the server doesn't support search.
   538	var ErrNoSearchRoot = errors.New("client: server doesn't support search")
   539	
   540	// ErrNoHelpRoot is returned by HelpRoot if the server doesn't have a help handler.
   541	var ErrNoHelpRoot = errors.New("client: server does not have a help handler")
   542	
   543	// ErrNoShareRoot is returned by ShareRoot if the server doesn't have a share handler.
   544	var ErrNoShareRoot = errors.New("client: server does not have a share handler")
   545	
   546	// ErrNoSigning is returned by ServerKeyID if the server doesn't support signing.
   547	var ErrNoSigning = fmt.Errorf("client: server doesn't support signing")
   548	
   549	// ErrNoStorageGeneration is returned by StorageGeneration if the
   550	// server doesn't report a storage generation value.
   551	var ErrNoStorageGeneration = errors.New("client: server doesn't report a storage generation")
   552	
   553	// ErrNoSync is returned by SyncHandlers if the server does not advertise syncs.
   554	var ErrNoSync = errors.New("client: server has no sync handlers")
   555	
   556	// BlobRoot returns the server's blobroot URL prefix.
   557	// If the client was constructed with an explicit path,
   558	// that path is used. Otherwise the server's
   559	// default advertised blobRoot is used.
   560	func (c *Client) BlobRoot() (string, error) {
   561		prefix, err := c.prefix()
   562		if err != nil {
   563			return "", err
   564		}
   565		return prefix + "/", nil
   566	}
   567	
   568	// ServerKeyID returns the server's GPG public key ID, in its long (16 capital
   569	// hex digits) format. If the server isn't running a sign handler, the error
   570	// will be ErrNoSigning.
   571	func (c *Client) ServerKeyID() (string, error) {
   572		if err := c.condDiscovery(); err != nil {
   573			return "", err
   574		}
   575		if c.serverKeyID == "" {
   576			return "", ErrNoSigning
   577		}
   578		return c.serverKeyID, nil
   579	}
   580	
   581	// ServerPublicKeyBlobRef returns the server's public key blobRef
   582	// If the server isn't running a sign handler, the error will be ErrNoSigning.
   583	func (c *Client) ServerPublicKeyBlobRef() (blob.Ref, error) {
   584		if err := c.condDiscovery(); err != nil {
   585			return blob.Ref{}, err
   586		}
   587	
   588		if !c.serverPublicKeyBlobRef.Valid() {
   589			return blob.Ref{}, ErrNoSigning
   590		}
   591		return c.serverPublicKeyBlobRef, nil
   592	}
   593	
   594	// SearchRoot returns the server's search handler.
   595	// If the server isn't running an index and search handler, the error
   596	// will be ErrNoSearchRoot.
   597	func (c *Client) SearchRoot() (string, error) {
   598		if err := c.condDiscovery(); err != nil {
   599			return "", err
   600		}
   601		if c.searchRoot == "" {
   602			return "", ErrNoSearchRoot
   603		}
   604		return c.searchRoot, nil
   605	}
   606	
   607	// HelpRoot returns the server's help handler.
   608	// If the server isn't running a help handler, the error will be
   609	// ErrNoHelpRoot.
   610	func (c *Client) HelpRoot() (string, error) {
   611		if err := c.condDiscovery(); err != nil {
   612			return "", err
   613		}
   614		if c.helpRoot == "" {
   615			return "", ErrNoHelpRoot
   616		}
   617		return c.helpRoot, nil
   618	}
   619	
   620	// ShareRoot returns the server's share handler prefix URL.
   621	// If the server isn't running a share handler, the error will be
   622	// ErrNoShareRoot.
   623	func (c *Client) ShareRoot() (string, error) {
   624		if err := c.condDiscovery(); err != nil {
   625			return "", err
   626		}
   627		if c.shareRoot == "" {
   628			return "", ErrNoShareRoot
   629		}
   630		return c.shareRoot, nil
   631	}
   632	
   633	// SignHandler returns the server's sign handler.
   634	// If the server isn't running a sign handler, the error will be
   635	// ErrNoSigning.
   636	func (c *Client) SignHandler() (string, error) {
   637		if err := c.condDiscovery(); err != nil {
   638			return "", err
   639		}
   640		if c.signHandler == "" {
   641			return "", ErrNoSigning
   642		}
   643		return c.signHandler, nil
   644	}
   645	
   646	// StorageGeneration returns the server's unique ID for its storage
   647	// generation, reset whenever storage is reset, moved, or partially
   648	// lost.
   649	//
   650	// This is a value that can be used in client cache keys to add
   651	// certainty that they're talking to the same instance as previously.
   652	//
   653	// If the server doesn't return such a value, the error will be
   654	// ErrNoStorageGeneration.
   655	func (c *Client) StorageGeneration() (string, error) {
   656		if err := c.condDiscovery(); err != nil {
   657			return "", err
   658		}
   659		if c.storageGen == "" {
   660			return "", ErrNoStorageGeneration
   661		}
   662		return c.storageGen, nil
   663	}
   664	
   665	// HasLegacySHA1 reports whether the server has SHA-1 blobs indexed.
   666	func (c *Client) HasLegacySHA1() (bool, error) {
   667		if err := c.condDiscovery(); err != nil {
   668			return false, err
   669		}
   670		return c.hasLegacySHA1, nil
   671	}
   672	
   673	// SyncInfo holds the data that were acquired with a discovery
   674	// and that are relevant to a syncHandler.
   675	type SyncInfo struct {
   676		From    string
   677		To      string
   678		ToIndex bool // whether this sync is from a blob storage to an index
   679	}
   680	
   681	// SyncHandlers returns the server's sync handlers "from" and
   682	// "to" prefix URLs.
   683	// If the server isn't running any sync handler, the error
   684	// will be ErrNoSync.
   685	func (c *Client) SyncHandlers() ([]*SyncInfo, error) {
   686		if err := c.condDiscovery(); err != nil {
   687			return nil, err
   688		}
   689		if c.syncHandlers == nil {
   690			return nil, ErrNoSync
   691		}
   692		return c.syncHandlers, nil
   693	}
   694	
   695	var _ search.GetRecentPermanoder = (*Client)(nil)
   696	
   697	// GetRecentPermanodes implements search.GetRecentPermanoder against a remote server over HTTP.
   698	func (c *Client) GetRecentPermanodes(ctx context.Context, req *search.RecentRequest) (*search.RecentResponse, error) {
   699		sr, err := c.SearchRoot()
   700		if err != nil {
   701			return nil, err
   702		}
   703		url := sr + req.URLSuffix()
   704		hreq := c.newRequest(ctx, "GET", url)
   705		hres, err := c.expect2XX(hreq)
   706		if err != nil {
   707			return nil, err
   708		}
   709		res := new(search.RecentResponse)
   710		if err := httputil.DecodeJSON(hres, res); err != nil {
   711			return nil, err
   712		}
   713		if err := res.Err(); err != nil {
   714			return nil, err
   715		}
   716		return res, nil
   717	}
   718	
   719	// GetPermanodesWithAttr searches for permanodes that match the given search request.
   720	// The Fuzzy option in the request must not be set, and the Attribute option
   721	// must be set.
   722	// Only indexed attributes may be queried.
   723	func (c *Client) GetPermanodesWithAttr(ctx context.Context, req *search.WithAttrRequest) (*search.WithAttrResponse, error) {
   724		sr, err := c.SearchRoot()
   725		if err != nil {
   726			return nil, err
   727		}
   728		url := sr + req.URLSuffix()
   729		hreq := c.newRequest(ctx, "GET", url)
   730		hres, err := c.expect2XX(hreq)
   731		if err != nil {
   732			return nil, err
   733		}
   734		res := new(search.WithAttrResponse)
   735		if err := httputil.DecodeJSON(hres, res); err != nil {
   736			return nil, err
   737		}
   738		if err := res.Err(); err != nil {
   739			return nil, err
   740		}
   741		return res, nil
   742	}
   743	
   744	func (c *Client) Describe(ctx context.Context, req *search.DescribeRequest) (*search.DescribeResponse, error) {
   745		sr, err := c.SearchRoot()
   746		if err != nil {
   747			return nil, err
   748		}
   749		url := sr + req.URLSuffixPost()
   750		body, err := json.MarshalIndent(req, "", "\t")
   751		if err != nil {
   752			return nil, err
   753		}
   754		hreq := c.newRequest(ctx, "POST", url, bytes.NewReader(body))
   755		hres, err := c.expect2XX(hreq)
   756		if err != nil {
   757			return nil, err
   758		}
   759		res := new(search.DescribeResponse)
   760		if err := httputil.DecodeJSON(hres, res); err != nil {
   761			return nil, err
   762		}
   763		return res, nil
   764	}
   765	
   766	func (c *Client) GetClaims(ctx context.Context, req *search.ClaimsRequest) (*search.ClaimsResponse, error) {
   767		sr, err := c.SearchRoot()
   768		if err != nil {
   769			return nil, err
   770		}
   771		url := sr + req.URLSuffix()
   772		hreq := c.newRequest(ctx, "GET", url)
   773		hres, err := c.expect2XX(hreq)
   774		if err != nil {
   775			return nil, err
   776		}
   777		res := new(search.ClaimsResponse)
   778		if err := httputil.DecodeJSON(hres, res); err != nil {
   779			return nil, err
   780		}
   781		return res, nil
   782	}
   783	
   784	func (c *Client) query(ctx context.Context, req *search.SearchQuery) (*http.Response, error) {
   785		sr, err := c.SearchRoot()
   786		if err != nil {
   787			return nil, err
   788		}
   789		url := sr + req.URLSuffix()
   790		body, err := json.Marshal(req)
   791		if err != nil {
   792			return nil, err
   793		}
   794		hreq := c.newRequest(ctx, "POST", url, bytes.NewReader(body))
   795		return c.expect2XX(hreq)
   796	}
   797	
   798	func (c *Client) Query(ctx context.Context, req *search.SearchQuery) (*search.SearchResult, error) {
   799		hres, err := c.query(ctx, req)
   800		if err != nil {
   801			return nil, err
   802		}
   803		res := new(search.SearchResult)
   804		if err := httputil.DecodeJSON(hres, res); err != nil {
   805			return nil, err
   806		}
   807		return res, nil
   808	}
   809	
   810	// QueryRaw sends req and returns the body of the response, which should be the
   811	// unparsed JSON of a search.SearchResult.
   812	func (c *Client) QueryRaw(ctx context.Context, req *search.SearchQuery) ([]byte, error) {
   813		hres, err := c.query(ctx, req)
   814		if err != nil {
   815			return nil, err
   816		}
   817		defer hres.Body.Close()
   818		return ioutil.ReadAll(hres.Body)
   819	}
   820	
   821	// SearchExistingFileSchema does a search query looking for an
   822	// existing file with entire contents of wholeRef, then does a HEAD
   823	// request to verify the file still exists on the server. If so,
   824	// it returns that file schema's blobref.
   825	//
   826	// If multiple wholeRef values are provided, any may match. This is
   827	// used for searching for the file by multiple wholeRef hashes
   828	// (e.g. SHA-224 and SHA-1).
   829	//
   830	// It returns (zero, nil) if not found. A non-nil error is only returned
   831	// if there were problems searching.
   832	func (c *Client) SearchExistingFileSchema(ctx context.Context, wholeRef ...blob.Ref) (blob.Ref, error) {
   833		sr, err := c.SearchRoot()
   834		if err != nil {
   835			return blob.Ref{}, err
   836		}
   837		if len(wholeRef) == 0 {
   838			return blob.Ref{}, nil
   839		}
   840		url := sr + "camli/search/files"
   841		for i, ref := range wholeRef {
   842			if i == 0 {
   843				url += "?wholedigest=" + ref.String()
   844			} else {
   845				url += "&wholedigest=" + ref.String()
   846			}
   847		}
   848		req := c.newRequest(ctx, "GET", url)
   849		res, err := c.doReqGated(req)
   850		if err != nil {
   851			return blob.Ref{}, err
   852		}
   853		if res.StatusCode != 200 {
   854			body, _ := ioutil.ReadAll(io.LimitReader(res.Body, 1<<20))
   855			res.Body.Close()
   856			return blob.Ref{}, fmt.Errorf("client: got status code %d from URL %s; body %s", res.StatusCode, url, body)
   857		}
   858		var ress camtypes.FileSearchResponse
   859		if err := httputil.DecodeJSON(res, &ress); err != nil {
   860			// Check that we're not just hitting the change introduced in 2018-01-13-6e8a5930c9fee81640c6c75a9a549fec98064186
   861			mismatch, err := c.versionMismatch(ctx)
   862			if err != nil {
   863				log.Printf("Could not verify whether client is too recent or server is too old: %v", err)
   864			} else if mismatch {
   865				return blob.Ref{}, fmt.Errorf("Client is too recent for this server. Use a client built before 2018-01-13-6e8a5930c9, or upgrade the server to after that revision.")
   866			}
   867			return blob.Ref{}, fmt.Errorf("client: error parsing JSON from URL %s: %v", url, err)
   868		}
   869		if len(ress.Files) == 0 {
   870			return blob.Ref{}, nil
   871		}
   872		for wholeRef, files := range ress.Files {
   873			for _, f := range files {
   874				if c.FileHasContents(ctx, f, blob.MustParse(wholeRef)) {
   875					return f, nil
   876				}
   877			}
   878		}
   879		return blob.Ref{}, nil
   880	}
   881	
   882	// versionMismatch returns true if the server was built before 2018-01-13 and
   883	// the client was built at or after 2018-01-13.
   884	func (c *Client) versionMismatch(ctx context.Context) (bool, error) {
   885		const shortRFC3339 = "2006-01-02"
   886		version := buildinfo.GitInfo
   887		if version == "" {
   888			return false, errors.New("unknown client version")
   889		}
   890		version = version[:10] // keep only the date part
   891		clientDate, err := time.Parse(shortRFC3339, version)
   892		if err != nil {
   893			return false, fmt.Errorf("could not parse date from version %q: %v", version, err)
   894		}
   895		apiChangeDate, _ := time.Parse(shortRFC3339, "2018-01-13")
   896		if !clientDate.After(apiChangeDate) {
   897			// client is old enough, all good.
   898			return false, nil
   899		}
   900		url := c.discoRoot() + "/status/status.json"
   901		println(url)
   902		req := c.newRequest(ctx, "GET", url)
   903		res, err := c.doReqGated(req)
   904		if err != nil {
   905			return false, err
   906		}
   907		if res.StatusCode != 200 {
   908			body, _ := ioutil.ReadAll(io.LimitReader(res.Body, 1<<20))
   909			res.Body.Close()
   910			return false, fmt.Errorf("got status code %d from URL %s; body %s", res.StatusCode, url, body)
   911		}
   912		var status struct {
   913			Version string `json:"version"`
   914		}
   915		if err := httputil.DecodeJSON(res, &status); err != nil {
   916			return false, fmt.Errorf("error parsing JSON from URL %s: %v", url, err)
   917		}
   918		serverVersion := status.Version[:10]
   919		serverDate, err := time.Parse(shortRFC3339, serverVersion)
   920		if err != nil {
   921			return false, fmt.Errorf("could not parse date from server version %q: %v", status.Version, err)
   922		}
   923		if serverDate.After(apiChangeDate) {
   924			// server is recent enough, all good.
   925			return false, nil
   926		}
   927		return true, nil
   928	}
   929	
   930	// FileHasContents returns true iff f refers to a "file" or "bytes" schema blob,
   931	// the server is configured with a "download helper", and the server responds
   932	// that all chunks of 'f' are available and match the digest of wholeRef.
   933	func (c *Client) FileHasContents(ctx context.Context, f, wholeRef blob.Ref) bool {
   934		if err := c.condDiscovery(); err != nil {
   935			return false
   936		}
   937		if c.downloadHelper == "" {
   938			return false
   939		}
   940		req := c.newRequest(ctx, "HEAD", c.downloadHelper+f.String()+"/?verifycontents="+wholeRef.String())
   941		res, err := c.expect2XX(req)
   942		if err != nil {
   943			log.Printf("download helper HEAD error: %v", err)
   944			return false
   945		}
   946		defer res.Body.Close()
   947		return res.Header.Get("X-Camli-Contents") == wholeRef.String()
   948	}
   949	
   950	// prefix returns the URL prefix before "/camli/", or before
   951	// the blobref hash in case of a share URL.
   952	// Examples: http://foo.com:3179/bs or http://foo.com:3179/share
   953	func (c *Client) prefix() (string, error) {
   954		if err := c.prefixOnce.Do(c.initPrefix); err != nil {
   955			return "", err
   956		}
   957		return c.prefixv, nil
   958	}
   959	
   960	// blobPrefix returns the URL prefix before the blobref hash.
   961	// Example: http://foo.com:3179/bs/camli or http://foo.com:3179/share
   962	func (c *Client) blobPrefix() (string, error) {
   963		pfx, err := c.prefix()
   964		if err != nil {
   965			return "", err
   966		}
   967		if !c.isSharePrefix {
   968			pfx += "/camli"
   969		}
   970		return pfx, nil
   971	}
   972	
   973	// discoRoot returns the user defined server for this client. It prepends "https://" if no scheme was specified.
   974	func (c *Client) discoRoot() string {
   975		s := c.server
   976		if c.sameOrigin {
   977			s = strings.TrimPrefix(s, "http://")
   978			s = strings.TrimPrefix(s, "https://")
   979			parts := strings.SplitN(s, "/", 1)
   980			if len(parts) < 2 {
   981				return "/"
   982			}
   983			return "/" + parts[1]
   984		}
   985		if !strings.HasPrefix(s, "http") {
   986			s = "https://" + s
   987		}
   988		return s
   989	}
   990	
   991	// initPrefix uses the user provided server URL to define the URL
   992	// prefix to the blobserver root. If the server URL has a path
   993	// component then it is directly used, otherwise the blobRoot
   994	// from the discovery is used as the path.
   995	func (c *Client) initPrefix() error {
   996		c.isSharePrefix = false
   997		root := c.discoRoot()
   998		u, err := url.Parse(root)
   999		if err != nil {
  1000			return err
  1001		}
  1002		if len(u.Path) > 1 {
  1003			c.prefixv = strings.TrimRight(root, "/")
  1004			return nil
  1005		}
  1006		return c.condDiscovery()
  1007	}
  1008	
  1009	func (c *Client) condDiscovery() error {
  1010		if c.sto != nil {
  1011			return errors.New("client not using HTTP")
  1012		}
  1013		return c.discoOnce.Do(c.doDiscovery)
  1014	}
  1015	
  1016	// DiscoveryDoc returns the server's JSON discovery document.
  1017	// This method exists purely for the "camtool discovery" command.
  1018	// Clients shouldn't have to parse this themselves.
  1019	func (c *Client) DiscoveryDoc(ctx context.Context) (io.Reader, error) {
  1020		res, err := c.discoveryResp(ctx)
  1021		if err != nil {
  1022			return nil, err
  1023		}
  1024		defer res.Body.Close()
  1025		const maxSize = 1 << 20
  1026		all, err := ioutil.ReadAll(io.LimitReader(res.Body, maxSize+1))
  1027		if err != nil {
  1028			return nil, err
  1029		}
  1030		if len(all) > maxSize {
  1031			return nil, errors.New("discovery document oddly large")
  1032		}
  1033		if len(all) > 0 && all[len(all)-1] != '\n' {
  1034			all = append(all, '\n')
  1035		}
  1036		return bytes.NewReader(all), err
  1037	}
  1038	
  1039	// HTTPVersion reports the HTTP version in use, such as "HTTP/1.1" or "HTTP/2.0".
  1040	func (c *Client) HTTPVersion(ctx context.Context) (string, error) {
  1041		req := c.newRequest(ctx, "HEAD", c.discoRoot(), nil)
  1042		res, err := c.doReqGated(req)
  1043		if err != nil {
  1044			return "", err
  1045		}
  1046		return res.Proto, err
  1047	}
  1048	
  1049	func (c *Client) discoveryResp(ctx context.Context) (*http.Response, error) {
  1050		// If the path is just "" or "/", do discovery against
  1051		// the URL to see which path we should actually use.
  1052		req := c.newRequest(ctx, "GET", c.discoRoot(), nil)
  1053		req.Header.Set("Accept", "text/x-camli-configuration")
  1054		res, err := c.doReqGated(req)
  1055		if err != nil {
  1056			return nil, err
  1057		}
  1058		if res.StatusCode != 200 {
  1059			res.Body.Close()
  1060			errMsg := fmt.Sprintf("got status %q from blobserver URL %q during configuration discovery", res.Status, c.discoRoot())
  1061			if res.StatusCode == 401 && c.authErr != nil {
  1062				errMsg = fmt.Sprintf("%v. %v", c.authErr, errMsg)
  1063			}
  1064			return nil, errors.New(errMsg)
  1065		}
  1066		// TODO(bradfitz): little weird in retrospect that we request
  1067		// text/x-camli-configuration and expect to get back
  1068		// text/javascript.  Make them consistent.
  1069		if ct := res.Header.Get("Content-Type"); ct != "text/javascript" {
  1070			res.Body.Close()
  1071			return nil, fmt.Errorf("Blobserver returned unexpected type %q from discovery", ct)
  1072		}
  1073		return res, nil
  1074	}
  1075	
  1076	func (c *Client) doDiscovery() error {
  1077		ctx := context.TODO()
  1078		root, err := url.Parse(c.discoRoot())
  1079		if err != nil {
  1080			return err
  1081		}
  1082	
  1083		res, err := c.discoveryResp(ctx)
  1084		if err != nil {
  1085			return err
  1086		}
  1087	
  1088		var disco camtypes.Discovery
  1089		if err := httputil.DecodeJSON(res, &disco); err != nil {
  1090			return err
  1091		}
  1092	
  1093		if disco.SearchRoot == "" {
  1094			c.searchRoot = ""
  1095		} else {
  1096			u, err := root.Parse(disco.SearchRoot)
  1097			if err != nil {
  1098				return fmt.Errorf("client: invalid searchRoot %q; failed to resolve", disco.SearchRoot)
  1099			}
  1100			c.searchRoot = u.String()
  1101		}
  1102	
  1103		u, err := root.Parse(disco.HelpRoot)
  1104		if err != nil {
  1105			return fmt.Errorf("client: invalid helpRoot %q; failed to resolve", disco.HelpRoot)
  1106		}
  1107		c.helpRoot = u.String()
  1108	
  1109		u, err = root.Parse(disco.ShareRoot)
  1110		if err != nil {
  1111			return fmt.Errorf("client: invalid shareRoot %q; failed to resolve", disco.ShareRoot)
  1112		}
  1113		c.shareRoot = u.String()
  1114	
  1115		c.storageGen = disco.StorageGeneration
  1116		c.hasLegacySHA1 = disco.HasLegacySHA1Index
  1117	
  1118		u, err = root.Parse(disco.BlobRoot)
  1119		if err != nil {
  1120			return fmt.Errorf("client: error resolving blobRoot: %v", err)
  1121		}
  1122		c.prefixv = strings.TrimRight(u.String(), "/")
  1123	
  1124		if disco.UIDiscovery != nil {
  1125			u, err = root.Parse(disco.DownloadHelper)
  1126			if err != nil {
  1127				return fmt.Errorf("client: invalid downloadHelper %q; failed to resolve", disco.DownloadHelper)
  1128			}
  1129			c.downloadHelper = u.String()
  1130		}
  1131	
  1132		if disco.SyncHandlers != nil {
  1133			for _, v := range disco.SyncHandlers {
  1134				ufrom, err := root.Parse(v.From)
  1135				if err != nil {
  1136					return fmt.Errorf("client: invalid %q \"from\" sync; failed to resolve", v.From)
  1137				}
  1138				uto, err := root.Parse(v.To)
  1139				if err != nil {
  1140					return fmt.Errorf("client: invalid %q \"to\" sync; failed to resolve", v.To)
  1141				}
  1142				c.syncHandlers = append(c.syncHandlers, &SyncInfo{
  1143					From:    ufrom.String(),
  1144					To:      uto.String(),
  1145					ToIndex: v.ToIndex,
  1146				})
  1147			}
  1148		}
  1149	
  1150		if disco.Signing != nil {
  1151			c.serverKeyID = disco.Signing.PublicKeyID
  1152			c.serverPublicKeyBlobRef = disco.Signing.PublicKeyBlobRef
  1153			c.signHandler = disco.Signing.SignHandler
  1154		}
  1155		return nil
  1156	}
  1157	
  1158	// GetJSON sends a GET request to url, and unmarshals the returned
  1159	// JSON response into data. The URL's host must match the client's
  1160	// configured server.
  1161	func (c *Client) GetJSON(ctx context.Context, url string, data interface{}) error {
  1162		if !strings.HasPrefix(url, c.discoRoot()) {
  1163			return fmt.Errorf("wrong URL (%q) for this server", url)
  1164		}
  1165		hreq := c.newRequest(ctx, "GET", url)
  1166		resp, err := c.expect2XX(hreq)
  1167		if err != nil {
  1168			return err
  1169		}
  1170		return httputil.DecodeJSON(resp, data)
  1171	}
  1172	
  1173	// Post is like http://golang.org/pkg/net/http/#Client.Post
  1174	// but with implementation details like gated requests. The
  1175	// URL's host must match the client's configured server.
  1176	func (c *Client) Post(ctx context.Context, url string, bodyType string, body io.Reader) error {
  1177		resp, err := c.post(ctx, url, bodyType, body)
  1178		if err != nil {
  1179			return err
  1180		}
  1181		return resp.Body.Close()
  1182	}
  1183	
  1184	// Sign sends a request to the sign handler on server to sign the contents of r,
  1185	// and return them signed. It uses the same implementation details, such as gated
  1186	// requests, as Post.
  1187	func (c *Client) Sign(ctx context.Context, server string, r io.Reader) (signed []byte, err error) {
  1188		signHandler, err := c.SignHandler()
  1189		if err != nil {
  1190			return nil, err
  1191		}
  1192		signServer := strings.TrimSuffix(server, "/") + signHandler
  1193		resp, err := c.post(ctx, signServer, "application/x-www-form-urlencoded", r)
  1194		if err != nil {
  1195			return nil, err
  1196		}
  1197		defer resp.Body.Close()
  1198		return ioutil.ReadAll(resp.Body)
  1199	}
  1200	
  1201	func (c *Client) post(ctx context.Context, url string, bodyType string, body io.Reader) (*http.Response, error) {
  1202		if !c.sameOrigin && !strings.HasPrefix(url, c.discoRoot()) {
  1203			return nil, fmt.Errorf("wrong URL (%q) for this server", url)
  1204		}
  1205		req := c.newRequest(ctx, "POST", url, body)
  1206		req.Header.Set("Content-Type", bodyType)
  1207		res, err := c.expect2XX(req)
  1208		if err != nil {
  1209			return nil, err
  1210		}
  1211		return res, nil
  1212	}
  1213	
  1214	// newRequest creates a request with the authentication header, and with the
  1215	// appropriate scheme and port in the case of self-signed TLS.
  1216	func (c *Client) newRequest(ctx context.Context, method, url string, body ...io.Reader) *http.Request {
  1217		var bodyR io.Reader
  1218		if len(body) > 0 {
  1219			bodyR = body[0]
  1220		}
  1221		if len(body) > 1 {
  1222			panic("too many body arguments")
  1223		}
  1224		req, err := http.NewRequest(method, url, bodyR)
  1225		if err != nil {
  1226			panic(err.Error())
  1227		}
  1228		// not done by http.NewRequest in Go 1.0:
  1229		if br, ok := bodyR.(*bytes.Reader); ok {
  1230			req.ContentLength = int64(br.Len())
  1231		}
  1232		c.authMode.AddAuthHeader(req)
  1233		return req.WithContext(ctx)
  1234	}
  1235	
  1236	// expect2XX will doReqGated and promote HTTP response codes outside of
  1237	// the 200-299 range to a non-nil error containing the response body.
  1238	func (c *Client) expect2XX(req *http.Request) (*http.Response, error) {
  1239		res, err := c.doReqGated(req)
  1240		if err == nil && (res.StatusCode < 200 || res.StatusCode > 299) {
  1241			buf := new(bytes.Buffer)
  1242			io.CopyN(buf, res.Body, 1<<20)
  1243			res.Body.Close()
  1244			return res, fmt.Errorf("client: got status code %d from URL %s; body %s", res.StatusCode, req.URL.String(), buf.String())
  1245		}
  1246		return res, err
  1247	}
  1248	
  1249	func (c *Client) doReqGated(req *http.Request) (*http.Response, error) {
  1250		c.httpGate.Start()
  1251		defer c.httpGate.Done()
  1252		return c.httpClient.Do(req)
  1253	}
  1254	
  1255	// DialFunc returns the adequate dial function when we're on android.
  1256	func (c *Client) DialFunc() func(network, addr string) (net.Conn, error) {
  1257		if c.useTLS() {
  1258			return nil
  1259		}
  1260		if android.IsChild() {
  1261			return func(network, addr string) (net.Conn, error) {
  1262				return android.Dial(network, addr)
  1263			}
  1264		}
  1265		return nil
  1266	}
  1267	
  1268	func (c *Client) http2DialTLSFunc() func(network, addr string, cfg *tls.Config) (net.Conn, error) {
  1269		trustedCerts := c.getTrustedCerts()
  1270		if !c.insecureAnyTLSCert && len(trustedCerts) == 0 {
  1271			// TLS with normal/full verification.
  1272			// nil means http2 uses its default dialer.
  1273			return nil
  1274		}
  1275		return func(network, addr string, cfg *tls.Config) (net.Conn, error) {
  1276			// we own cfg, so we can mutate it:
  1277			cfg.InsecureSkipVerify = true
  1278			conn, err := tls.Dial(network, addr, cfg)
  1279			if err != nil {
  1280				return nil, err
  1281			}
  1282			if c.insecureAnyTLSCert {
  1283				return conn, err
  1284			}
  1285			state := conn.ConnectionState()
  1286			if p := state.NegotiatedProtocol; p != http2.NextProtoTLS {
  1287				return nil, fmt.Errorf("http2: unexpected ALPN protocol %q; want %q", p, http2.NextProtoTLS)
  1288			}
  1289			if !state.NegotiatedProtocolIsMutual {
  1290				return nil, errors.New("http2: could not negotiate protocol mutually")
  1291			}
  1292			certs := state.PeerCertificates
  1293			if len(certs) < 1 {
  1294				return nil, fmt.Errorf("no TLS peer certificates from %s", addr)
  1295			}
  1296			sig := hashutil.SHA256Prefix(certs[0].Raw)
  1297			for _, v := range trustedCerts {
  1298				if v == sig {
  1299					return conn, nil
  1300				}
  1301			}
  1302			return nil, fmt.Errorf("TLS server at %v presented untrusted certificate (signature %q)", addr, sig)
  1303		}
  1304	}
  1305	
  1306	// DialTLSFunc returns the adequate dial function, when using SSL, depending on
  1307	// whether we're using insecure TLS (certificate verification is disabled), or we
  1308	// have some trusted certs, or we're on android.
  1309	// If the client's config has some trusted certs, the server's certificate will
  1310	// be checked against those in the config after the TLS handshake.
  1311	func (c *Client) DialTLSFunc() func(network, addr string) (net.Conn, error) {
  1312		if !c.useTLS() {
  1313			return nil
  1314		}
  1315		trustedCerts := c.getTrustedCerts()
  1316		var stdTLS bool
  1317		if !c.insecureAnyTLSCert && len(trustedCerts) == 0 {
  1318			// TLS with normal/full verification.
  1319			stdTLS = true
  1320			if !android.IsChild() {
  1321				// Not android, so let the stdlib deal with it
  1322				return nil
  1323			}
  1324		}
  1325	
  1326		return func(network, addr string) (net.Conn, error) {
  1327			var conn *tls.Conn
  1328			var err error
  1329			if android.IsChild() {
  1330				ac, err := android.Dial(network, addr)
  1331				if err != nil {
  1332					return nil, err
  1333				}
  1334				var tlsConfig *tls.Config
  1335				if stdTLS {
  1336					tlsConfig, err = android.TLSConfig()
  1337					if err != nil {
  1338						return nil, err
  1339					}
  1340				} else {
  1341					tlsConfig = &tls.Config{InsecureSkipVerify: true}
  1342				}
  1343				// Since we're doing the TLS handshake ourselves, we need to set the ServerName,
  1344				// in case the server uses SNI (as is the case if it's relying on Let's Encrypt,
  1345				// for example).
  1346				tlsConfig.ServerName = c.serverNameOfAddr(addr)
  1347				conn = tls.Client(ac, tlsConfig)
  1348				if err := conn.Handshake(); err != nil {
  1349					return nil, err
  1350				}
  1351				if stdTLS {
  1352					// Normal TLS verification succeeded and we do not have
  1353					// additional trusted certificate fingerprints to check for.
  1354					return conn, nil
  1355				}
  1356			} else {
  1357				conn, err = tls.Dial(network, addr, &tls.Config{InsecureSkipVerify: true})
  1358				if err != nil {
  1359					return nil, err
  1360				}
  1361			}
  1362			if c.insecureAnyTLSCert {
  1363				return conn, nil
  1364			}
  1365			certs := conn.ConnectionState().PeerCertificates
  1366			if len(certs) < 1 {
  1367				return nil, fmt.Errorf("no TLS peer certificates from %s", addr)
  1368			}
  1369			sig := hashutil.SHA256Prefix(certs[0].Raw)
  1370			for _, v := range trustedCerts {
  1371				if v == sig {
  1372					return conn, nil
  1373				}
  1374			}
  1375			return nil, fmt.Errorf("TLS server at %v presented untrusted certificate (signature %q)", addr, sig)
  1376		}
  1377	}
  1378	
  1379	// serverNameOfAddr returns the host part of addr, or the empty string if addr
  1380	// is not a valid address (see net.Dial). Additionally, if host is an IP literal,
  1381	// serverNameOfAddr returns the empty string.
  1382	func (c *Client) serverNameOfAddr(addr string) string {
  1383		serverName, _, err := net.SplitHostPort(addr)
  1384		if err != nil {
  1385			c.printf("could not get server name from address %q: %v", addr, err)
  1386			return ""
  1387		}
  1388		if ip := net.ParseIP(serverName); ip != nil {
  1389			return ""
  1390		}
  1391		return serverName
  1392	}
  1393	
  1394	// Signer returns the client's Signer, if any. The Signer signs JSON
  1395	// mutation claims.
  1396	func (c *Client) Signer() (*schema.Signer, error) {
  1397		c.signerOnce.Do(c.signerInit)
  1398		return c.signer, c.signerErr
  1399	}
  1400	
  1401	func (c *Client) signerInit() {
  1402		c.signer, c.signerErr = c.buildSigner()
  1403	}
  1404	
  1405	func (c *Client) buildSigner() (*schema.Signer, error) {
  1406		c.initSignerPublicKeyBlobrefOnce.Do(c.initSignerPublicKeyBlobref)
  1407		if !c.signerPublicKeyRef.Valid() {
  1408			return nil, camtypes.ErrClientNoPublicKey
  1409		}
  1410		return schema.NewSigner(c.signerPublicKeyRef, strings.NewReader(c.publicKeyArmored), c.SecretRingFile())
  1411	}
  1412	
  1413	// sigTime optionally specifies the signature time.
  1414	// If zero, the current time is used.
  1415	func (c *Client) signBlob(ctx context.Context, bb schema.Buildable, sigTime time.Time) (string, error) {
  1416		signer, err := c.Signer()
  1417		if err != nil {
  1418			return "", err
  1419		}
  1420		return bb.Builder().SignAt(ctx, signer, sigTime)
  1421	}
  1422	
  1423	// uploadPublicKey uploads the public key (if one is defined), so
  1424	// subsequent (likely synchronous) indexing of uploaded signed blobs
  1425	// will have access to the public key to verify it. In the normal
  1426	// case, the stat cache prevents this from doing anything anyway.
  1427	func (c *Client) uploadPublicKey(ctx context.Context) error {
  1428		sigRef := c.SignerPublicKeyBlobref()
  1429		if !sigRef.Valid() {
  1430			return nil
  1431		}
  1432		var err error
  1433		if _, keyUploaded := c.haveCache.StatBlobCache(sigRef); !keyUploaded {
  1434			_, err = c.uploadString(ctx, c.publicKeyArmored, false)
  1435		}
  1436		return err
  1437	}
  1438	
  1439	// checkMatchingKeys compares the client's and the server's keys and logs if they differ.
  1440	func (c *Client) checkMatchingKeys() {
  1441		serverKey, err := c.ServerKeyID()
  1442		// The server provides the full (16 digit) key fingerprint but schema.Signer only stores
  1443		// the short (8 digit) key ID.
  1444		if err == nil && len(serverKey) >= 8 {
  1445			shortServerKey := serverKey[len(serverKey)-8:]
  1446			if shortServerKey != c.signer.KeyID() {
  1447				log.Printf("Warning: client (%s) and server (%s) keys differ.", c.signer.KeyID(), shortServerKey)
  1448			}
  1449		}
  1450	}
  1451	
  1452	func (c *Client) UploadAndSignBlob(ctx context.Context, b schema.AnyBlob) (*PutResult, error) {
  1453		signed, err := c.signBlob(ctx, b.Blob(), time.Time{})
  1454		if err != nil {
  1455			return nil, err
  1456		}
  1457		c.checkMatchingKeys()
  1458		if err := c.uploadPublicKey(ctx); err != nil {
  1459			return nil, err
  1460		}
  1461		return c.uploadString(ctx, signed, false)
  1462	}
  1463	
  1464	func (c *Client) UploadBlob(ctx context.Context, b schema.AnyBlob) (*PutResult, error) {
  1465		// TODO(bradfitz): ask the blob for its own blobref, rather
  1466		// than changing the hash function with uploadString?
  1467		return c.uploadString(ctx, b.Blob().JSON(), true)
  1468	}
  1469	
  1470	func (c *Client) uploadString(ctx context.Context, s string, stat bool) (*PutResult, error) {
  1471		uh := NewUploadHandleFromString(s)
  1472		uh.SkipStat = !stat
  1473		return c.Upload(ctx, uh)
  1474	}
  1475	
  1476	func (c *Client) UploadNewPermanode(ctx context.Context) (*PutResult, error) {
  1477		unsigned := schema.NewUnsignedPermanode()
  1478		return c.UploadAndSignBlob(ctx, unsigned)
  1479	}
  1480	
  1481	func (c *Client) UploadPlannedPermanode(ctx context.Context, key string, sigTime time.Time) (*PutResult, error) {
  1482		unsigned := schema.NewPlannedPermanode(key)
  1483		signed, err := c.signBlob(ctx, unsigned, sigTime)
  1484		if err != nil {
  1485			return nil, err
  1486		}
  1487		c.checkMatchingKeys()
  1488		if err := c.uploadPublicKey(ctx); err != nil {
  1489			return nil, err
  1490		}
  1491		return c.uploadString(ctx, signed, true)
  1492	}
  1493	
  1494	// IsIgnoredFile returns whether the file at fullpath should be ignored by pk-put.
  1495	// The fullpath is checked against the ignoredFiles list, trying the following rules in this order:
  1496	// 1) star-suffix style matching (.e.g *.jpg).
  1497	// 2) Shell pattern match as done by http://golang.org/pkg/path/filepath/#Match
  1498	// 3) If the pattern is an absolute path to a directory, fullpath matches if it is that directory or a child of it.
  1499	// 4) If the pattern is a relative path, fullpath matches if it has pattern as a path component (i.e the pattern is a part of fullpath that fits exactly between two path separators).
  1500	func (c *Client) IsIgnoredFile(fullpath string) bool {
  1501		c.initIgnoredFilesOnce.Do(c.initIgnoredFiles)
  1502		return c.ignoreChecker(fullpath)
  1503	}
  1504	
  1505	// Close closes the client.
  1506	func (c *Client) Close() error {
  1507		if cl, ok := c.sto.(io.Closer); ok {
  1508			return cl.Close()
  1509		}
  1510		if c := c.HTTPClient(); c != nil {
  1511			switch t := c.Transport.(type) {
  1512			case *http.Transport:
  1513				t.CloseIdleConnections()
  1514			case *http2.Transport:
  1515				t.CloseIdleConnections()
  1516			}
  1517		}
  1518		return nil
  1519	}
  1520	
  1521	func httpGateSize(rt http.RoundTripper) int {
  1522		switch v := rt.(type) {
  1523		case *httputil.StatsTransport:
  1524			return httpGateSize(v.Transport)
  1525		case *http.Transport:
  1526			return maxParallelHTTP_h1
  1527		case *http2.Transport:
  1528			return maxParallelHTTP_h2
  1529		default:
  1530			return maxParallelHTTP_h1 // conservative default
  1531		}
  1532	}
Website layout inspired by memcached.
Content by the authors.