1from __future__ import unicode_literals 2 3import re 4from schema import Or, Optional 5 6from dvc.exceptions import DvcException 7from dvc.utils.compat import str 8from dvc.remote.base import RemoteBase 9 10 11class OutputDoesNotExistError(DvcException): 12 def __init__(self, path): 13 msg = "output '{}' does not exist".format(path) 14 super(OutputDoesNotExistError, self).__init__(msg) 15 16 17class OutputIsNotFileOrDirError(DvcException): 18 def __init__(self, path): 19 msg = "output '{}' is not a file or directory".format(path) 20 super(OutputIsNotFileOrDirError, self).__init__(msg) 21 22 23class OutputAlreadyTrackedError(DvcException): 24 def __init__(self, path): 25 msg = "output '{}' is already tracked by scm (e.g. git)".format(path) 26 super(OutputAlreadyTrackedError, self).__init__(msg) 27 28 29class OutputBase(object): 30 IS_DEPENDENCY = False 31 32 REMOTE = RemoteBase 33 34 PARAM_PATH = "path" 35 PARAM_CACHE = "cache" 36 PARAM_METRIC = "metric" 37 PARAM_METRIC_TYPE = "type" 38 PARAM_METRIC_XPATH = "xpath" 39 40 METRIC_SCHEMA = Or( 41 None, 42 bool, 43 { 44 Optional(PARAM_METRIC_TYPE): Or(str, None), 45 Optional(PARAM_METRIC_XPATH): Or(str, None), 46 }, 47 ) 48 49 DoesNotExistError = OutputDoesNotExistError 50 IsNotFileOrDirError = OutputIsNotFileOrDirError 51 52 def __init__( 53 self, stage, path, info=None, remote=None, cache=True, metric=False 54 ): 55 self.stage = stage 56 self.repo = stage.repo 57 self.url = path 58 self.info = info 59 self.remote = remote or self.REMOTE(self.repo, {}) 60 self.use_cache = False if self.IS_DEPENDENCY else cache 61 self.metric = False if self.IS_DEPENDENCY else metric 62 63 if ( 64 self.use_cache 65 and getattr(self.repo.cache, self.REMOTE.scheme) is None 66 ): 67 raise DvcException( 68 "no cache location setup for '{}' outputs.".format( 69 self.REMOTE.scheme 70 ) 71 ) 72 73 def __repr__(self): 74 return "{class_name}: '{url}'".format( 75 class_name=type(self).__name__, url=(self.url or "No url") 76 ) 77 78 def __str__(self): 79 return self.url 80 81 @property 82 def is_local(self): 83 raise DvcException( 84 "is local is not supported for {}".format(self.scheme) 85 ) 86 87 @classmethod 88 def match(cls, url): 89 return re.match(cls.REMOTE.REGEX, url) 90 91 def group(self, name): 92 match = self.match(self.url) 93 if not match: 94 return None 95 return match.group(name) 96 97 @classmethod 98 def supported(cls, url): 99 return cls.match(url) is not None 100 101 @property 102 def scheme(self): 103 return self.REMOTE.scheme 104 105 @property 106 def path(self): 107 return self.path_info["path"] 108 109 @property 110 def sep(self): 111 return "/" 112 113 @property 114 def checksum(self): 115 return self.info.get(self.remote.PARAM_CHECKSUM) 116 117 @property 118 def exists(self): 119 return self.remote.exists(self.path_info) 120 121 def changed_checksum(self): 122 return ( 123 self.checksum 124 != self.remote.save_info(self.path_info)[ 125 self.remote.PARAM_CHECKSUM 126 ] 127 ) 128 129 def changed_cache(self): 130 if not self.use_cache or not self.checksum: 131 return True 132 133 cache = self.repo.cache.__getattribute__(self.scheme) 134 135 return cache.changed_cache(self.checksum) 136 137 def status(self): 138 if self.checksum and self.use_cache and self.changed_cache(): 139 return {str(self): "not in cache"} 140 141 if not self.exists: 142 return {str(self): "deleted"} 143 144 if self.changed_checksum(): 145 return {str(self): "modified"} 146 147 if not self.checksum: 148 return {str(self): "new"} 149 150 return {} 151 152 def changed(self): 153 return bool(self.status()) 154 155 def save(self): 156 if not self.exists: 157 raise self.DoesNotExistError(self) 158 159 self.info = self.remote.save_info(self.path_info) 160 161 def commit(self): 162 if self.use_cache: 163 getattr(self.repo.cache, self.scheme).save( 164 self.path_info, self.info 165 ) 166 167 def dumpd(self): 168 ret = self.info.copy() 169 ret[self.PARAM_PATH] = self.url 170 171 if self.IS_DEPENDENCY: 172 return ret 173 174 ret[self.PARAM_CACHE] = self.use_cache 175 176 if isinstance(self.metric, dict): 177 if ( 178 self.PARAM_METRIC_XPATH in self.metric 179 and not self.metric[self.PARAM_METRIC_XPATH] 180 ): 181 del self.metric[self.PARAM_METRIC_XPATH] 182 183 ret[self.PARAM_METRIC] = self.metric 184 185 return ret 186 187 def verify_metric(self): 188 raise DvcException( 189 "verify metric is not supported for {}".format(self.scheme) 190 ) 191 192 def download(self, to_info, resume=False): 193 self.remote.download([self.path_info], [to_info], resume=resume) 194 195 def checkout(self, force=False): 196 if not self.use_cache: 197 return 198 199 getattr(self.repo.cache, self.scheme).checkout( 200 self.path_info, self.info, force=force 201 ) 202 203 def remove(self, ignore_remove=False): 204 self.remote.remove(self.path_info) 205 if self.scheme != "local": 206 return 207 208 if ignore_remove and self.use_cache and self.is_local: 209 self.repo.scm.ignore_remove(self.path) 210 211 def move(self, out): 212 if self.scheme == "local" and self.use_cache and self.is_local: 213 self.repo.scm.ignore_remove(self.path) 214 215 self.remote.move(self.path_info, out.path_info) 216 self.url = out.url 217 self.path_info = out.path_info 218 self.save() 219 self.commit() 220 221 if self.scheme == "local" and self.use_cache and self.is_local: 222 self.repo.scm.ignore(self.path) 223