Home Download Docs Code Community
     1	/*
     2	Copyright 2014 The Perkeep Authors
     3	
     4	Licensed under the Apache License, Version 2.0 (the "License");
     5	you may not use this file except in compliance with the License.
     6	You may obtain a copy of the License at
     7	
     8	     http://www.apache.org/licenses/LICENSE-2.0
     9	
    10	Unless required by applicable law or agreed to in writing, software
    11	distributed under the License is distributed on an "AS IS" BASIS,
    12	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13	See the License for the specific language governing permissions and
    14	limitations under the License.
    15	*/
    16	
    17	package blob
    18	
    19	import (
    20		"bytes"
    21		"context"
    22		"fmt"
    23		"io"
    24		"io/ioutil"
    25		"sync/atomic"
    26		"unicode/utf8"
    27	
    28		"perkeep.org/pkg/constants"
    29	)
    30	
    31	// Blob represents a blob. Use the methods Size, SizedRef and
    32	// ReadAll to query and get data from Blob.
    33	type Blob struct {
    34		ref     Ref
    35		size    uint32
    36		readAll func(context.Context) ([]byte, error)
    37		mem     atomic.Value // of []byte, if in memory
    38	}
    39	
    40	// NewBlob constructs a Blob from its Ref, size and a function that
    41	// returns the contents of the blob. Any error in the function newReader when
    42	// constructing the io.ReadCloser should be returned upon the first call to Read or
    43	// Close.
    44	func NewBlob(ref Ref, size uint32, readAll func(ctx context.Context) ([]byte, error)) *Blob {
    45		return &Blob{
    46			ref:     ref,
    47			size:    size,
    48			readAll: readAll,
    49		}
    50	}
    51	
    52	// Size returns the size of the blob (in bytes).
    53	func (b *Blob) Size() uint32 {
    54		return b.size
    55	}
    56	
    57	// SizedRef returns the SizedRef corresponding to the blob.
    58	func (b *Blob) SizedRef() SizedRef {
    59		return SizedRef{b.ref, b.size}
    60	}
    61	
    62	// Ref returns the blob's reference.
    63	func (b *Blob) Ref() Ref { return b.ref }
    64	
    65	// ReadAll reads the blob completely to memory, using the provided
    66	// context, and then returns a reader over it. The Reader will not
    67	// have errors except EOF.
    68	//
    69	// The provided is only ctx is used until ReadAll returns.
    70	func (b *Blob) ReadAll(ctx context.Context) (*bytes.Reader, error) {
    71		mem, err := b.getMem(ctx)
    72		if err != nil {
    73			return nil, err
    74		}
    75		return bytes.NewReader(mem), nil
    76	}
    77	
    78	func (b *Blob) getMem(ctx context.Context) ([]byte, error) {
    79		mem, ok := b.mem.Load().([]byte)
    80		if ok {
    81			return mem, nil
    82		}
    83		mem, err := b.readAll(ctx)
    84		if err != nil {
    85			return nil, err
    86		}
    87		if uint32(len(mem)) != b.size {
    88			return nil, fmt.Errorf("Blob.ReadAll read %d bytes of %v; expected %d", len(mem), b.ref, b.size)
    89		}
    90		b.mem.Store(mem)
    91		return mem, nil
    92	}
    93	
    94	// ValidContents reports whether the hash of blob's content matches
    95	// its reference.
    96	func (b *Blob) ValidContents(ctx context.Context) error {
    97		mem, err := b.getMem(ctx)
    98		if err != nil {
    99			return err
   100		}
   101		h := b.ref.Hash()
   102		h.Write(mem)
   103		if !b.ref.HashMatches(h) {
   104			return fmt.Errorf("blob contents don't match digest for ref %v", b.ref)
   105		}
   106		return nil
   107	}
   108	
   109	// IsUTF8 reports whether the blob is entirely UTF-8.
   110	func (b *Blob) IsUTF8(ctx context.Context) (bool, error) {
   111		mem, err := b.getMem(ctx)
   112		if err != nil {
   113			return false, err
   114		}
   115		return utf8.Valid(mem), nil
   116	}
   117	
   118	// A reader reads a blob's contents.
   119	// It adds a no-op Close method to a *bytes.Reader.
   120	type reader struct {
   121		*bytes.Reader
   122	}
   123	
   124	func (reader) Close() error { return nil }
   125	
   126	// FromFetcher fetches br from fetcher and slurps its contents to
   127	// memory. It does not validate the blob's digest.  Use the
   128	// Blob.ValidContents method for that.
   129	func FromFetcher(ctx context.Context, fetcher Fetcher, br Ref) (*Blob, error) {
   130		rc, size, err := fetcher.Fetch(ctx, br)
   131		if err != nil {
   132			return nil, err
   133		}
   134		defer rc.Close()
   135		return FromReader(ctx, br, rc, size)
   136	}
   137	
   138	// FromReader slurps the given blob from r to memory.
   139	// It does not validate the blob's digest.  Use the
   140	// Blob.ValidContents method for that.
   141	func FromReader(ctx context.Context, br Ref, r io.Reader, size uint32) (*Blob, error) {
   142		if size > constants.MaxBlobSize {
   143			return nil, fmt.Errorf("blob: %v with reported size %d is over max size of %d", br, size, constants.MaxBlobSize)
   144		}
   145		buf := make([]byte, size)
   146		// TODO: use ctx here during ReadFull? add context-checking Reader wrapper?
   147		if n, err := io.ReadFull(r, buf); err != nil {
   148			return nil, fmt.Errorf("blob: after reading %d bytes of %v: %v", n, br, err)
   149		}
   150		n, _ := io.CopyN(ioutil.Discard, r, 1)
   151		if n > 0 {
   152			return nil, fmt.Errorf("blob: %v had more than reported %d bytes", br, size)
   153		}
   154		b := NewBlob(br, uint32(size), func(context.Context) ([]byte, error) {
   155			return buf, nil
   156		})
   157		b.mem.Store(buf)
   158		return b, nil
   159	}
Website layout inspired by memcached.
Content by the authors.