1// Copyright (C) 2019 Storj Labs, Inc. 2// See LICENSE for copying information. 3 4package metainfo 5 6import ( 7 "fmt" 8 "strconv" 9 "strings" 10 "time" 11 12 "storj.io/common/memory" 13 "storj.io/storj/satellite/metabase/segmentloop" 14 "storj.io/storj/satellite/metainfo/piecedeletion" 15) 16 17const ( 18 // BoltPointerBucket is the string representing the bucket used for `PointerEntries` in BoltDB. 19 BoltPointerBucket = "pointers" 20) 21 22// RSConfig is a configuration struct that keeps details about default 23// redundancy strategy information. 24// 25// Can be used as a flag. 26type RSConfig struct { 27 ErasureShareSize memory.Size 28 Min int 29 Repair int 30 Success int 31 Total int 32} 33 34// Type implements pflag.Value. 35func (RSConfig) Type() string { return "metainfo.RSConfig" } 36 37// String is required for pflag.Value. 38func (rs *RSConfig) String() string { 39 return fmt.Sprintf("%d/%d/%d/%d-%s", 40 rs.Min, 41 rs.Repair, 42 rs.Success, 43 rs.Total, 44 rs.ErasureShareSize.String()) 45} 46 47// Set sets the value from a string in the format k/m/o/n-size (min/repair/optimal/total-erasuresharesize). 48func (rs *RSConfig) Set(s string) error { 49 // Split on dash. Expect two items. First item is RS numbers. Second item is memory.Size. 50 info := strings.Split(s, "-") 51 if len(info) != 2 { 52 return Error.New("Invalid default RS config (expect format k/m/o/n-ShareSize, got %s)", s) 53 } 54 rsNumbersString := info[0] 55 shareSizeString := info[1] 56 57 // Attempt to parse "-size" part of config. 58 shareSizeInt, err := memory.ParseString(shareSizeString) 59 if err != nil { 60 return Error.New("Invalid share size in RS config: '%s', %w", shareSizeString, err) 61 } 62 shareSize := memory.Size(shareSizeInt) 63 64 // Split on forward slash. Expect exactly four positive non-decreasing integers. 65 rsNumbers := strings.Split(rsNumbersString, "/") 66 if len(rsNumbers) != 4 { 67 return Error.New("Invalid default RS numbers (wrong size, expect 4): %s", rsNumbersString) 68 } 69 70 minValue := 1 71 values := []int{} 72 for _, nextValueString := range rsNumbers { 73 nextValue, err := strconv.Atoi(nextValueString) 74 if err != nil { 75 return Error.New("Invalid default RS numbers (should all be valid integers): %s, %w", rsNumbersString, err) 76 } 77 if nextValue < minValue { 78 return Error.New("Invalid default RS numbers (should be non-decreasing): %s", rsNumbersString) 79 } 80 values = append(values, nextValue) 81 minValue = nextValue 82 } 83 84 rs.ErasureShareSize = shareSize 85 rs.Min = values[0] 86 rs.Repair = values[1] 87 rs.Success = values[2] 88 rs.Total = values[3] 89 90 return nil 91} 92 93// RateLimiterConfig is a configuration struct for endpoint rate limiting. 94type RateLimiterConfig struct { 95 Enabled bool `help:"whether rate limiting is enabled." releaseDefault:"true" devDefault:"true"` 96 Rate float64 `help:"request rate per project per second." releaseDefault:"1000" devDefault:"100" testDefault:"1000"` 97 CacheCapacity int `help:"number of projects to cache." releaseDefault:"10000" devDefault:"10" testDefault:"100"` 98 CacheExpiration time.Duration `help:"how long to cache the projects limiter." releaseDefault:"10m" devDefault:"10s"` 99} 100 101// ProjectLimitConfig is a configuration struct for default project limits. 102type ProjectLimitConfig struct { 103 MaxBuckets int `help:"max bucket count for a project." default:"100" testDefault:"10"` 104} 105 106// Config is a configuration struct that is everything you need to start a metainfo. 107type Config struct { 108 DatabaseURL string `help:"the database connection string to use" default:"postgres://"` 109 MinRemoteSegmentSize memory.Size `default:"1240" testDefault:"0" help:"minimum remote segment size"` // TODO: fix tests to work with 1024 110 MaxInlineSegmentSize memory.Size `default:"4KiB" help:"maximum inline segment size"` 111 // we have such default value because max value for ObjectKey is 1024(1 Kib) but EncryptedObjectKey 112 // has encryption overhead 16 bytes. So overall size is 1024 + 16 * 16. 113 MaxEncryptedObjectKeyLength int `default:"1280" help:"maximum encrypted object key length"` 114 MaxSegmentSize memory.Size `default:"64MiB" help:"maximum segment size"` 115 MaxMetadataSize memory.Size `default:"2KiB" help:"maximum segment metadata size"` 116 MaxCommitInterval time.Duration `default:"48h" testDefault:"1h" help:"maximum time allowed to pass between creating and committing a segment"` 117 MinPartSize memory.Size `default:"5MiB" testDefault:"0" help:"minimum allowed part size (last part has no minimum size limit)"` 118 MaxNumberOfParts int `default:"10000" help:"maximum number of parts object can contain"` 119 Overlay bool `default:"true" help:"toggle flag if overlay is enabled"` 120 RS RSConfig `releaseDefault:"29/35/80/110-256B" devDefault:"4/6/8/10-256B" help:"redundancy scheme configuration in the format k/m/o/n-sharesize"` 121 SegmentLoop segmentloop.Config `help:"segment loop configuration"` 122 RateLimiter RateLimiterConfig `help:"rate limiter configuration"` 123 ProjectLimits ProjectLimitConfig `help:"project limit configuration"` 124 PieceDeletion piecedeletion.Config `help:"piece deletion configuration"` 125} 126