1# cython: language_level=3
2
3import os
4import re
5import stat
6import subprocess
7
8from ..helpers import workarounds
9from ..helpers import posix_acl_use_stored_uid_gid
10from ..helpers import user2uid, group2gid
11from ..helpers import safe_decode, safe_encode
12from .base import SyncFile as BaseSyncFile
13from .base import safe_fadvise
14from .posix import swidth
15try:
16    from .syncfilerange import sync_file_range, SYNC_FILE_RANGE_WRITE, SYNC_FILE_RANGE_WAIT_BEFORE, SYNC_FILE_RANGE_WAIT_AFTER
17    SYNC_FILE_RANGE_LOADED = True
18except ImportError:
19    SYNC_FILE_RANGE_LOADED = False
20
21from libc cimport errno
22from libc.stdint cimport int64_t
23
24API_VERSION = '1.1_04'
25
26cdef extern from "sys/types.h":
27    int ACL_TYPE_ACCESS
28    int ACL_TYPE_DEFAULT
29
30cdef extern from "sys/acl.h":
31    ctypedef struct _acl_t:
32        pass
33    ctypedef _acl_t *acl_t
34
35    int acl_free(void *obj)
36    acl_t acl_get_file(const char *path, int type)
37    int acl_set_file(const char *path, int type, acl_t acl)
38    acl_t acl_from_text(const char *buf)
39    char *acl_to_text(acl_t acl, ssize_t *len)
40
41cdef extern from "acl/libacl.h":
42    int acl_extended_file(const char *path)
43
44cdef extern from "linux/fs.h":
45    # ioctls
46    int FS_IOC_SETFLAGS
47    int FS_IOC_GETFLAGS
48
49    # inode flags
50    int FS_NODUMP_FL
51    int FS_IMMUTABLE_FL
52    int FS_APPEND_FL
53    int FS_COMPR_FL
54
55cdef extern from "sys/ioctl.h":
56    int ioctl(int fildes, int request, ...)
57
58cdef extern from "unistd.h":
59    int _SC_PAGESIZE
60    long sysconf(int name)
61
62cdef extern from "string.h":
63    char *strerror(int errnum)
64
65_comment_re = re.compile(' *#.*', re.M)
66
67
68BSD_TO_LINUX_FLAGS = {
69    stat.UF_NODUMP: FS_NODUMP_FL,
70    stat.UF_IMMUTABLE: FS_IMMUTABLE_FL,
71    stat.UF_APPEND: FS_APPEND_FL,
72    stat.UF_COMPRESSED: FS_COMPR_FL,
73}
74
75
76def set_flags(path, bsd_flags, fd=None):
77    if fd is None:
78        st = os.stat(path, follow_symlinks=False)
79        if stat.S_ISBLK(st.st_mode) or stat.S_ISCHR(st.st_mode) or stat.S_ISLNK(st.st_mode):
80            # see comment in get_flags()
81            return
82    cdef int flags = 0
83    for bsd_flag, linux_flag in BSD_TO_LINUX_FLAGS.items():
84        if bsd_flags & bsd_flag:
85            flags |= linux_flag
86    open_fd = fd is None
87    if open_fd:
88        fd = os.open(path, os.O_RDONLY|os.O_NONBLOCK|os.O_NOFOLLOW)
89    try:
90        if ioctl(fd, FS_IOC_SETFLAGS, &flags) == -1:
91            error_number = errno.errno
92            if error_number != errno.EOPNOTSUPP:
93                raise OSError(error_number, strerror(error_number).decode(), path)
94    finally:
95        if open_fd:
96            os.close(fd)
97
98
99def get_flags(path, st):
100    if stat.S_ISBLK(st.st_mode) or stat.S_ISCHR(st.st_mode) or stat.S_ISLNK(st.st_mode):
101        # avoid opening devices files - trying to open non-present devices can be rather slow.
102        # avoid opening symlinks, O_NOFOLLOW would make the open() fail anyway.
103        return 0
104    cdef int linux_flags
105    try:
106        fd = os.open(path, os.O_RDONLY|os.O_NONBLOCK|os.O_NOFOLLOW)
107    except OSError:
108        return 0
109    try:
110        if ioctl(fd, FS_IOC_GETFLAGS, &linux_flags) == -1:
111            return 0
112    finally:
113        os.close(fd)
114    bsd_flags = 0
115    for bsd_flag, linux_flag in BSD_TO_LINUX_FLAGS.items():
116        if linux_flags & linux_flag:
117            bsd_flags |= bsd_flag
118    return bsd_flags
119
120
121def acl_use_local_uid_gid(acl):
122    """Replace the user/group field with the local uid/gid if possible
123    """
124    entries = []
125    for entry in safe_decode(acl).split('\n'):
126        if entry:
127            fields = entry.split(':')
128            if fields[0] == 'user' and fields[1]:
129                fields[1] = str(user2uid(fields[1], fields[3]))
130            elif fields[0] == 'group' and fields[1]:
131                fields[1] = str(group2gid(fields[1], fields[3]))
132            entries.append(':'.join(fields[:3]))
133    return safe_encode('\n'.join(entries))
134
135
136cdef acl_append_numeric_ids(acl):
137    """Extend the "POSIX 1003.1e draft standard 17" format with an additional uid/gid field
138    """
139    entries = []
140    for entry in _comment_re.sub('', safe_decode(acl)).split('\n'):
141        if entry:
142            type, name, permission = entry.split(':')
143            if name and type == 'user':
144                entries.append(':'.join([type, name, permission, str(user2uid(name, name))]))
145            elif name and type == 'group':
146                entries.append(':'.join([type, name, permission, str(group2gid(name, name))]))
147            else:
148                entries.append(entry)
149    return safe_encode('\n'.join(entries))
150
151
152cdef acl_numeric_ids(acl):
153    """Replace the "POSIX 1003.1e draft standard 17" user/group field with uid/gid
154    """
155    entries = []
156    for entry in _comment_re.sub('', safe_decode(acl)).split('\n'):
157        if entry:
158            type, name, permission = entry.split(':')
159            if name and type == 'user':
160                uid = str(user2uid(name, name))
161                entries.append(':'.join([type, uid, permission, uid]))
162            elif name and type == 'group':
163                gid = str(group2gid(name, name))
164                entries.append(':'.join([type, gid, permission, gid]))
165            else:
166                entries.append(entry)
167    return safe_encode('\n'.join(entries))
168
169
170def acl_get(path, item, st, numeric_owner=False):
171    cdef acl_t default_acl = NULL
172    cdef acl_t access_acl = NULL
173    cdef char *default_text = NULL
174    cdef char *access_text = NULL
175
176    p = <bytes>os.fsencode(path)
177    if stat.S_ISLNK(st.st_mode) or acl_extended_file(p) <= 0:
178        return
179    if numeric_owner:
180        converter = acl_numeric_ids
181    else:
182        converter = acl_append_numeric_ids
183    try:
184        access_acl = acl_get_file(p, ACL_TYPE_ACCESS)
185        if access_acl:
186            access_text = acl_to_text(access_acl, NULL)
187            if access_text:
188                item['acl_access'] = converter(access_text)
189        default_acl = acl_get_file(p, ACL_TYPE_DEFAULT)
190        if default_acl:
191            default_text = acl_to_text(default_acl, NULL)
192            if default_text:
193                item['acl_default'] = converter(default_text)
194    finally:
195        acl_free(default_text)
196        acl_free(default_acl)
197        acl_free(access_text)
198        acl_free(access_acl)
199
200
201def acl_set(path, item, numeric_owner=False):
202    cdef acl_t access_acl = NULL
203    cdef acl_t default_acl = NULL
204
205    if stat.S_ISLNK(item.get('mode', 0)):
206        # Linux does not support setting ACLs on symlinks
207        return
208
209    p = <bytes>os.fsencode(path)
210    if numeric_owner:
211        converter = posix_acl_use_stored_uid_gid
212    else:
213        converter = acl_use_local_uid_gid
214    access_text = item.get('acl_access')
215    default_text = item.get('acl_default')
216    if access_text:
217        try:
218            access_acl = acl_from_text(<bytes>converter(access_text))
219            if access_acl:
220                acl_set_file(p, ACL_TYPE_ACCESS, access_acl)
221        finally:
222            acl_free(access_acl)
223    if default_text:
224        try:
225            default_acl = acl_from_text(<bytes>converter(default_text))
226            if default_acl:
227                acl_set_file(p, ACL_TYPE_DEFAULT, default_acl)
228        finally:
229            acl_free(default_acl)
230
231
232cdef _sync_file_range(fd, offset, length, flags):
233    assert offset & PAGE_MASK == 0, "offset %d not page-aligned" % offset
234    assert length & PAGE_MASK == 0, "length %d not page-aligned" % length
235    if sync_file_range(fd, offset, length, flags) != 0:
236        raise OSError(errno.errno, os.strerror(errno.errno))
237    safe_fadvise(fd, offset, length, 'DONTNEED')
238
239
240cdef unsigned PAGE_MASK = sysconf(_SC_PAGESIZE) - 1
241
242
243if 'basesyncfile' in workarounds or not SYNC_FILE_RANGE_LOADED:
244    class SyncFile(BaseSyncFile):
245        # if we are on platforms with a broken or not implemented sync_file_range,
246        # use the more generic BaseSyncFile to avoid issues.
247        # see basesyncfile description in our docs for details.
248        pass
249else:
250    # a real Linux, so we can do better. :)
251    class SyncFile(BaseSyncFile):
252        """
253        Implemented using sync_file_range for asynchronous write-out and fdatasync for actual durability.
254
255        "write-out" means that dirty pages (= data that was written) are submitted to an I/O queue and will be send to
256        disk in the immediate future.
257        """
258
259        def __init__(self, path, binary=False):
260            super().__init__(path, binary)
261            self.offset = 0
262            self.write_window = (16 * 1024 ** 2) & ~PAGE_MASK
263            self.last_sync = 0
264            self.pending_sync = None
265
266        def write(self, data):
267            self.offset += self.fd.write(data)
268            offset = self.offset & ~PAGE_MASK
269            if offset >= self.last_sync + self.write_window:
270                self.fd.flush()
271                _sync_file_range(self.fileno, self.last_sync, offset - self.last_sync, SYNC_FILE_RANGE_WRITE)
272                if self.pending_sync is not None:
273                    _sync_file_range(self.fileno, self.pending_sync, self.last_sync - self.pending_sync,
274                                     SYNC_FILE_RANGE_WRITE | SYNC_FILE_RANGE_WAIT_BEFORE | SYNC_FILE_RANGE_WAIT_AFTER)
275                self.pending_sync = self.last_sync
276                self.last_sync = offset
277
278        def sync(self):
279            self.fd.flush()
280            os.fdatasync(self.fileno)
281            # tell the OS that it does not need to cache what we just wrote,
282            # avoids spoiling the cache for the OS and other processes.
283            safe_fadvise(self.fileno, 0, 0, 'DONTNEED')
284