1	/*
     2	Copyright 2011 The Perkeep Authors
     3	
     4	Licensed under the Apache License, Version 2.0 (the "License");
     5	you may not use this file except in compliance with the License.
     6	You may obtain a copy of the License at
     7	
     8	     http://www.apache.org/licenses/LICENSE-2.0
     9	
    10	Unless required by applicable law or agreed to in writing, software
    11	distributed under the License is distributed on an "AS IS" BASIS,
    12	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13	See the License for the specific language governing permissions and
    14	limitations under the License.
    15	*/
    16	
    17	/*
    18	Package localdisk registers the "filesystem" blobserver storage type,
    19	storing blobs in a forest of sharded directories at the specified root.
    20	
    21	Example low-level config:
    22	
    23		"/storage/": {
    24		    "handler": "storage-filesystem",
    25		    "handlerArgs": {
    26		       "path": "/var/camlistore/blobs"
    27		     }
    28		},
    29	*/
    30	package localdisk // import "perkeep.org/pkg/blobserver/localdisk"
    31	
    32	import (
    33		"bytes"
    34		"fmt"
    35		"log"
    36		"os"
    37		"path/filepath"
    38	
    39		"perkeep.org/internal/osutil"
    40		"perkeep.org/pkg/blob"
    41		"perkeep.org/pkg/blobserver"
    42		"perkeep.org/pkg/blobserver/files"
    43		"perkeep.org/pkg/blobserver/local"
    44	
    45		"go4.org/jsonconfig"
    46		"go4.org/syncutil"
    47	)
    48	
    49	// TODO: rename DiskStorage to Storage.
    50	
    51	// DiskStorage implements the blobserver.Storage interface using the
    52	// local filesystem.
    53	type DiskStorage struct {
    54		blobserver.Storage
    55		blob.SubFetcher
    56	
    57		root string
    58	
    59		// gen will be nil if partition != ""
    60		gen *local.Generationer
    61	}
    62	
    63	// Validate we implement expected interfaces.
    64	var (
    65		_ blobserver.Storage = (*DiskStorage)(nil)
    66		_ blob.SubFetcher    = (*DiskStorage)(nil) // for blobpacked; Issue 1136
    67	)
    68	
    69	func (ds *DiskStorage) String() string {
    70		return fmt.Sprintf("\"filesystem\" file-per-blob at %s", ds.root)
    71	}
    72	
    73	// IsDir reports whether root is a localdisk (file-per-blob) storage directory.
    74	func IsDir(root string) (bool, error) {
    75		if osutil.DirExists(filepath.Join(root, "sha1")) {
    76			return true, nil
    77		}
    78		if osutil.DirExists(filepath.Join(root, blob.RefFromString("").HashName())) {
    79			return true, nil
    80		}
    81		return false, nil
    82	}
    83	
    84	const (
    85		// We refuse to create a DiskStorage when the user's ulimit is lower than
    86		// minFDLimit. 100 is ridiculously low, but the default value on OSX is 256, and we
    87		// don't want to fail by default, so our min value has to be lower than 256.
    88		minFDLimit         = 100
    89		recommendedFDLimit = 1024
    90	)
    91	
    92	// New returns a new local disk storage implementation at the provided
    93	// root directory, which must already exist.
    94	func New(root string) (*DiskStorage, error) {
    95		// Local disk.
    96		fi, err := os.Stat(root)
    97		if os.IsNotExist(err) {
    98			// As a special case, we auto-created the "packed" directory for subpacked.
    99			if filepath.Base(root) == "packed" {
   100				if err := os.Mkdir(root, 0700); err != nil {
   101					return nil, fmt.Errorf("failed to mkdir packed directory: %v", err)
   102				}
   103				fi, err = os.Stat(root)
   104			} else {
   105				return nil, fmt.Errorf("Storage root %q doesn't exist", root)
   106			}
   107		}
   108		if err != nil {
   109			return nil, fmt.Errorf("Failed to stat directory %q: %v", root, err)
   110		}
   111		if !fi.IsDir() {
   112			return nil, fmt.Errorf("storage root %q exists but is not a directory", root)
   113		}
   114		fileSto := files.NewStorage(files.OSFS(), root)
   115		ds := &DiskStorage{
   116			Storage:    fileSto,
   117			SubFetcher: fileSto,
   118			root:       root,
   119			gen:        local.NewGenerationer(root),
   120		}
   121		if _, _, err := ds.StorageGeneration(); err != nil {
   122			return nil, fmt.Errorf("Error initialization generation for %q: %v", root, err)
   123		}
   124		ul, err := osutil.MaxFD()
   125		if err != nil {
   126			if err == osutil.ErrNotSupported {
   127				// Do not set the gate on Windows, since we don't know the ulimit.
   128				return ds, nil
   129			}
   130			return nil, err
   131		}
   132		if ul < minFDLimit {
   133			return nil, fmt.Errorf("the max number of open file descriptors on your system (ulimit -n) is too low. Please fix it with 'ulimit -S -n X' with X being at least %d", recommendedFDLimit)
   134		}
   135		// Setting the gate to 80% of the ulimit, to leave a bit of room for other file ops happening in Perkeep.
   136		// TODO(mpl): make this used and enforced Perkeep-wide. Issue #837.
   137		fileSto.SetNewFileGate(syncutil.NewGate(int(ul * 80 / 100)))
   138	
   139		err = ds.checkFS()
   140		if err != nil {
   141			return nil, err
   142		}
   143		return ds, nil
   144	}
   145	
   146	func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) {
   147		path := config.RequiredString("path")
   148		if err := config.Validate(); err != nil {
   149			return nil, err
   150		}
   151		return New(path)
   152	}
   153	
   154	func init() {
   155		blobserver.RegisterStorageConstructor("filesystem", blobserver.StorageConstructor(newFromConfig))
   156	}
   157	
   158	// checkFS verifies the DiskStorage root storage path
   159	// operations include: stat, read/write file, mkdir, delete (files and directories)
   160	//
   161	// TODO: move this into the files package too?
   162	func (ds *DiskStorage) checkFS() (ret error) {
   163		tempdir, err := os.MkdirTemp(ds.root, "")
   164		if err != nil {
   165			return fmt.Errorf("localdisk check: unable to create tempdir in %s, err=%v", ds.root, err)
   166		}
   167		defer func() {
   168			err := os.RemoveAll(tempdir)
   169			if err != nil {
   170				cleanErr := fmt.Errorf("localdisk check: unable to clean temp dir: %v", err)
   171				if ret == nil {
   172					ret = cleanErr
   173				} else {
   174					log.Printf("WARNING: %v", cleanErr)
   175				}
   176			}
   177		}()
   178	
   179		tempfile := filepath.Join(tempdir, "FILE.tmp")
   180		filename := filepath.Join(tempdir, "FILE")
   181		data := []byte("perkeep rocks")
   182		err = os.WriteFile(tempfile, data, 0644)
   183		if err != nil {
   184			return fmt.Errorf("localdisk check: unable to write into %s, err=%v", ds.root, err)
   185		}
   186	
   187		out, err := os.ReadFile(tempfile)
   188		if err != nil {
   189			return fmt.Errorf("localdisk check: unable to read from %s, err=%v", tempfile, err)
   190		}
   191		if bytes.Compare(out, data) != 0 {
   192			return fmt.Errorf("localdisk check: tempfile contents didn't match, got=%q", out)
   193		}
   194		if _, err := os.Lstat(filename); !os.IsNotExist(err) {
   195			return fmt.Errorf("localdisk check: didn't expect file to exist, Lstat had other error, err=%v", err)
   196		}
   197		if err := os.Rename(tempfile, filename); err != nil {
   198			return fmt.Errorf("localdisk check: rename failed, err=%v", err)
   199		}
   200		if _, err := os.Lstat(filename); err != nil {
   201			return fmt.Errorf("localdisk check: after rename passed Lstat had error, err=%v", err)
   202		}
   203		return nil
   204	}