1	/*
     2	Copyright 2014 The Perkeep Authors
     3	
     4	Licensed under the Apache License, Version 2.0 (the "License");
     5	you may not use this file except in compliance with the License.
     6	You may obtain a copy of the License at
     7	
     8	     http://www.apache.org/licenses/LICENSE-2.0
     9	
    10	Unless required by applicable law or agreed to in writing, software
    11	distributed under the License is distributed on an "AS IS" BASIS,
    12	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13	See the License for the specific language governing permissions and
    14	limitations under the License.
    15	*/
    16	
    17	package blob
    18	
    19	import (
    20		"bytes"
    21		"context"
    22		"fmt"
    23		"io"
    24		"sync/atomic"
    25		"unicode/utf8"
    26	
    27		"perkeep.org/pkg/constants"
    28	)
    29	
    30	// Blob represents a blob. Use the methods Size, SizedRef and
    31	// ReadAll to query and get data from Blob.
    32	type Blob struct {
    33		ref     Ref
    34		size    uint32
    35		readAll func(context.Context) ([]byte, error)
    36		mem     atomic.Value // of []byte, if in memory
    37	}
    38	
    39	// NewBlob constructs a Blob from its Ref, size and a function that
    40	// returns the contents of the blob. Any error in the function newReader when
    41	// constructing the io.ReadCloser should be returned upon the first call to Read or
    42	// Close.
    43	func NewBlob(ref Ref, size uint32, readAll func(ctx context.Context) ([]byte, error)) *Blob {
    44		return &Blob{
    45			ref:     ref,
    46			size:    size,
    47			readAll: readAll,
    48		}
    49	}
    50	
    51	// Size returns the size of the blob (in bytes).
    52	func (b *Blob) Size() uint32 {
    53		return b.size
    54	}
    55	
    56	// SizedRef returns the SizedRef corresponding to the blob.
    57	func (b *Blob) SizedRef() SizedRef {
    58		return SizedRef{b.ref, b.size}
    59	}
    60	
    61	// Ref returns the blob's reference.
    62	func (b *Blob) Ref() Ref { return b.ref }
    63	
    64	// ReadAll reads the blob completely to memory, using the provided
    65	// context, and then returns a reader over it. The Reader will not
    66	// have errors except EOF.
    67	//
    68	// The provided is only ctx is used until ReadAll returns.
    69	func (b *Blob) ReadAll(ctx context.Context) (*bytes.Reader, error) {
    70		mem, err := b.getMem(ctx)
    71		if err != nil {
    72			return nil, err
    73		}
    74		return bytes.NewReader(mem), nil
    75	}
    76	
    77	func (b *Blob) getMem(ctx context.Context) ([]byte, error) {
    78		mem, ok := b.mem.Load().([]byte)
    79		if ok {
    80			return mem, nil
    81		}
    82		mem, err := b.readAll(ctx)
    83		if err != nil {
    84			return nil, err
    85		}
    86		if uint32(len(mem)) != b.size {
    87			return nil, fmt.Errorf("Blob.ReadAll read %d bytes of %v; expected %d", len(mem), b.ref, b.size)
    88		}
    89		b.mem.Store(mem)
    90		return mem, nil
    91	}
    92	
    93	// ValidContents reports whether the hash of blob's content matches
    94	// its reference.
    95	func (b *Blob) ValidContents(ctx context.Context) error {
    96		mem, err := b.getMem(ctx)
    97		if err != nil {
    98			return err
    99		}
   100		h := b.ref.Hash()
   101		h.Write(mem)
   102		if !b.ref.HashMatches(h) {
   103			return fmt.Errorf("blob contents don't match digest for ref %v", b.ref)
   104		}
   105		return nil
   106	}
   107	
   108	// IsUTF8 reports whether the blob is entirely UTF-8.
   109	func (b *Blob) IsUTF8(ctx context.Context) (bool, error) {
   110		mem, err := b.getMem(ctx)
   111		if err != nil {
   112			return false, err
   113		}
   114		return utf8.Valid(mem), nil
   115	}
   116	
   117	// FromFetcher fetches br from fetcher and slurps its contents to
   118	// memory. It does not validate the blob's digest.  Use the
   119	// Blob.ValidContents method for that.
   120	func FromFetcher(ctx context.Context, fetcher Fetcher, br Ref) (*Blob, error) {
   121		rc, size, err := fetcher.Fetch(ctx, br)
   122		if err != nil {
   123			return nil, err
   124		}
   125		defer rc.Close()
   126		return FromReader(ctx, br, rc, size)
   127	}
   128	
   129	// FromReader slurps the given blob from r to memory.
   130	// It does not validate the blob's digest.  Use the
   131	// Blob.ValidContents method for that.
   132	func FromReader(ctx context.Context, br Ref, r io.Reader, size uint32) (*Blob, error) {
   133		if size > constants.MaxBlobSize {
   134			return nil, fmt.Errorf("blob: %v with reported size %d is over max size of %d", br, size, constants.MaxBlobSize)
   135		}
   136		buf := make([]byte, size)
   137		// TODO: use ctx here during ReadFull? add context-checking Reader wrapper?
   138		if n, err := io.ReadFull(r, buf); err != nil {
   139			return nil, fmt.Errorf("blob: after reading %d bytes of %v: %v", n, br, err)
   140		}
   141		n, _ := io.CopyN(io.Discard, r, 1)
   142		if n > 0 {
   143			return nil, fmt.Errorf("blob: %v had more than reported %d bytes", br, size)
   144		}
   145		b := NewBlob(br, uint32(size), func(context.Context) ([]byte, error) {
   146			return buf, nil
   147		})
   148		b.mem.Store(buf)
   149		return b, nil
   150	}