1# cython: language_level=3 2 3import os 4import re 5import stat 6import subprocess 7 8from ..helpers import workarounds 9from ..helpers import posix_acl_use_stored_uid_gid 10from ..helpers import user2uid, group2gid 11from ..helpers import safe_decode, safe_encode 12from .base import SyncFile as BaseSyncFile 13from .base import safe_fadvise 14from .posix import swidth 15try: 16 from .syncfilerange import sync_file_range, SYNC_FILE_RANGE_WRITE, SYNC_FILE_RANGE_WAIT_BEFORE, SYNC_FILE_RANGE_WAIT_AFTER 17 SYNC_FILE_RANGE_LOADED = True 18except ImportError: 19 SYNC_FILE_RANGE_LOADED = False 20 21from libc cimport errno 22from libc.stdint cimport int64_t 23 24API_VERSION = '1.1_04' 25 26cdef extern from "sys/types.h": 27 int ACL_TYPE_ACCESS 28 int ACL_TYPE_DEFAULT 29 30cdef extern from "sys/acl.h": 31 ctypedef struct _acl_t: 32 pass 33 ctypedef _acl_t *acl_t 34 35 int acl_free(void *obj) 36 acl_t acl_get_file(const char *path, int type) 37 int acl_set_file(const char *path, int type, acl_t acl) 38 acl_t acl_from_text(const char *buf) 39 char *acl_to_text(acl_t acl, ssize_t *len) 40 41cdef extern from "acl/libacl.h": 42 int acl_extended_file(const char *path) 43 44cdef extern from "linux/fs.h": 45 # ioctls 46 int FS_IOC_SETFLAGS 47 int FS_IOC_GETFLAGS 48 49 # inode flags 50 int FS_NODUMP_FL 51 int FS_IMMUTABLE_FL 52 int FS_APPEND_FL 53 int FS_COMPR_FL 54 55cdef extern from "sys/ioctl.h": 56 int ioctl(int fildes, int request, ...) 57 58cdef extern from "unistd.h": 59 int _SC_PAGESIZE 60 long sysconf(int name) 61 62cdef extern from "string.h": 63 char *strerror(int errnum) 64 65_comment_re = re.compile(' *#.*', re.M) 66 67 68BSD_TO_LINUX_FLAGS = { 69 stat.UF_NODUMP: FS_NODUMP_FL, 70 stat.UF_IMMUTABLE: FS_IMMUTABLE_FL, 71 stat.UF_APPEND: FS_APPEND_FL, 72 stat.UF_COMPRESSED: FS_COMPR_FL, 73} 74 75 76def set_flags(path, bsd_flags, fd=None): 77 if fd is None: 78 st = os.stat(path, follow_symlinks=False) 79 if stat.S_ISBLK(st.st_mode) or stat.S_ISCHR(st.st_mode) or stat.S_ISLNK(st.st_mode): 80 # see comment in get_flags() 81 return 82 cdef int flags = 0 83 for bsd_flag, linux_flag in BSD_TO_LINUX_FLAGS.items(): 84 if bsd_flags & bsd_flag: 85 flags |= linux_flag 86 open_fd = fd is None 87 if open_fd: 88 fd = os.open(path, os.O_RDONLY|os.O_NONBLOCK|os.O_NOFOLLOW) 89 try: 90 if ioctl(fd, FS_IOC_SETFLAGS, &flags) == -1: 91 error_number = errno.errno 92 if error_number != errno.EOPNOTSUPP: 93 raise OSError(error_number, strerror(error_number).decode(), path) 94 finally: 95 if open_fd: 96 os.close(fd) 97 98 99def get_flags(path, st): 100 if stat.S_ISBLK(st.st_mode) or stat.S_ISCHR(st.st_mode) or stat.S_ISLNK(st.st_mode): 101 # avoid opening devices files - trying to open non-present devices can be rather slow. 102 # avoid opening symlinks, O_NOFOLLOW would make the open() fail anyway. 103 return 0 104 cdef int linux_flags 105 try: 106 fd = os.open(path, os.O_RDONLY|os.O_NONBLOCK|os.O_NOFOLLOW) 107 except OSError: 108 return 0 109 try: 110 if ioctl(fd, FS_IOC_GETFLAGS, &linux_flags) == -1: 111 return 0 112 finally: 113 os.close(fd) 114 bsd_flags = 0 115 for bsd_flag, linux_flag in BSD_TO_LINUX_FLAGS.items(): 116 if linux_flags & linux_flag: 117 bsd_flags |= bsd_flag 118 return bsd_flags 119 120 121def acl_use_local_uid_gid(acl): 122 """Replace the user/group field with the local uid/gid if possible 123 """ 124 entries = [] 125 for entry in safe_decode(acl).split('\n'): 126 if entry: 127 fields = entry.split(':') 128 if fields[0] == 'user' and fields[1]: 129 fields[1] = str(user2uid(fields[1], fields[3])) 130 elif fields[0] == 'group' and fields[1]: 131 fields[1] = str(group2gid(fields[1], fields[3])) 132 entries.append(':'.join(fields[:3])) 133 return safe_encode('\n'.join(entries)) 134 135 136cdef acl_append_numeric_ids(acl): 137 """Extend the "POSIX 1003.1e draft standard 17" format with an additional uid/gid field 138 """ 139 entries = [] 140 for entry in _comment_re.sub('', safe_decode(acl)).split('\n'): 141 if entry: 142 type, name, permission = entry.split(':') 143 if name and type == 'user': 144 entries.append(':'.join([type, name, permission, str(user2uid(name, name))])) 145 elif name and type == 'group': 146 entries.append(':'.join([type, name, permission, str(group2gid(name, name))])) 147 else: 148 entries.append(entry) 149 return safe_encode('\n'.join(entries)) 150 151 152cdef acl_numeric_ids(acl): 153 """Replace the "POSIX 1003.1e draft standard 17" user/group field with uid/gid 154 """ 155 entries = [] 156 for entry in _comment_re.sub('', safe_decode(acl)).split('\n'): 157 if entry: 158 type, name, permission = entry.split(':') 159 if name and type == 'user': 160 uid = str(user2uid(name, name)) 161 entries.append(':'.join([type, uid, permission, uid])) 162 elif name and type == 'group': 163 gid = str(group2gid(name, name)) 164 entries.append(':'.join([type, gid, permission, gid])) 165 else: 166 entries.append(entry) 167 return safe_encode('\n'.join(entries)) 168 169 170def acl_get(path, item, st, numeric_owner=False): 171 cdef acl_t default_acl = NULL 172 cdef acl_t access_acl = NULL 173 cdef char *default_text = NULL 174 cdef char *access_text = NULL 175 176 p = <bytes>os.fsencode(path) 177 if stat.S_ISLNK(st.st_mode) or acl_extended_file(p) <= 0: 178 return 179 if numeric_owner: 180 converter = acl_numeric_ids 181 else: 182 converter = acl_append_numeric_ids 183 try: 184 access_acl = acl_get_file(p, ACL_TYPE_ACCESS) 185 if access_acl: 186 access_text = acl_to_text(access_acl, NULL) 187 if access_text: 188 item['acl_access'] = converter(access_text) 189 default_acl = acl_get_file(p, ACL_TYPE_DEFAULT) 190 if default_acl: 191 default_text = acl_to_text(default_acl, NULL) 192 if default_text: 193 item['acl_default'] = converter(default_text) 194 finally: 195 acl_free(default_text) 196 acl_free(default_acl) 197 acl_free(access_text) 198 acl_free(access_acl) 199 200 201def acl_set(path, item, numeric_owner=False): 202 cdef acl_t access_acl = NULL 203 cdef acl_t default_acl = NULL 204 205 if stat.S_ISLNK(item.get('mode', 0)): 206 # Linux does not support setting ACLs on symlinks 207 return 208 209 p = <bytes>os.fsencode(path) 210 if numeric_owner: 211 converter = posix_acl_use_stored_uid_gid 212 else: 213 converter = acl_use_local_uid_gid 214 access_text = item.get('acl_access') 215 default_text = item.get('acl_default') 216 if access_text: 217 try: 218 access_acl = acl_from_text(<bytes>converter(access_text)) 219 if access_acl: 220 acl_set_file(p, ACL_TYPE_ACCESS, access_acl) 221 finally: 222 acl_free(access_acl) 223 if default_text: 224 try: 225 default_acl = acl_from_text(<bytes>converter(default_text)) 226 if default_acl: 227 acl_set_file(p, ACL_TYPE_DEFAULT, default_acl) 228 finally: 229 acl_free(default_acl) 230 231 232cdef _sync_file_range(fd, offset, length, flags): 233 assert offset & PAGE_MASK == 0, "offset %d not page-aligned" % offset 234 assert length & PAGE_MASK == 0, "length %d not page-aligned" % length 235 if sync_file_range(fd, offset, length, flags) != 0: 236 raise OSError(errno.errno, os.strerror(errno.errno)) 237 safe_fadvise(fd, offset, length, 'DONTNEED') 238 239 240cdef unsigned PAGE_MASK = sysconf(_SC_PAGESIZE) - 1 241 242 243if 'basesyncfile' in workarounds or not SYNC_FILE_RANGE_LOADED: 244 class SyncFile(BaseSyncFile): 245 # if we are on platforms with a broken or not implemented sync_file_range, 246 # use the more generic BaseSyncFile to avoid issues. 247 # see basesyncfile description in our docs for details. 248 pass 249else: 250 # a real Linux, so we can do better. :) 251 class SyncFile(BaseSyncFile): 252 """ 253 Implemented using sync_file_range for asynchronous write-out and fdatasync for actual durability. 254 255 "write-out" means that dirty pages (= data that was written) are submitted to an I/O queue and will be send to 256 disk in the immediate future. 257 """ 258 259 def __init__(self, path, binary=False): 260 super().__init__(path, binary) 261 self.offset = 0 262 self.write_window = (16 * 1024 ** 2) & ~PAGE_MASK 263 self.last_sync = 0 264 self.pending_sync = None 265 266 def write(self, data): 267 self.offset += self.fd.write(data) 268 offset = self.offset & ~PAGE_MASK 269 if offset >= self.last_sync + self.write_window: 270 self.fd.flush() 271 _sync_file_range(self.fileno, self.last_sync, offset - self.last_sync, SYNC_FILE_RANGE_WRITE) 272 if self.pending_sync is not None: 273 _sync_file_range(self.fileno, self.pending_sync, self.last_sync - self.pending_sync, 274 SYNC_FILE_RANGE_WRITE | SYNC_FILE_RANGE_WAIT_BEFORE | SYNC_FILE_RANGE_WAIT_AFTER) 275 self.pending_sync = self.last_sync 276 self.last_sync = offset 277 278 def sync(self): 279 self.fd.flush() 280 os.fdatasync(self.fileno) 281 # tell the OS that it does not need to cache what we just wrote, 282 # avoids spoiling the cache for the OS and other processes. 283 safe_fadvise(self.fileno, 0, 0, 'DONTNEED') 284