1 /*
2 / Author: Sam Rushing <[email protected]>
3 / Hacked for Unix by AMK
4 / $Id$
5
6 / Modified to support mmap with offset - to map a 'window' of a file
7 / Author: Yotam Medini [email protected]
8 /
9 / mmapmodule.cpp -- map a view of a file into memory
10 /
11 / todo: need permission flags, perhaps a 'chsize' analog
12 / not all functions check range yet!!!
13 /
14 /
15 / This version of mmapmodule.c has been changed significantly
16 / from the original mmapfile.c on which it was based.
17 / The original version of mmapfile is maintained by Sam at
18 / ftp://squirl.nightmare.com/pub/python/python-ext.
19 */
20
21 #ifndef Py_BUILD_CORE_BUILTIN
22 # define Py_BUILD_CORE_MODULE 1
23 #endif
24
25 #define PY_SSIZE_T_CLEAN
26 #include <Python.h>
27 #include "pycore_bytesobject.h" // _PyBytes_Find()
28 #include "pycore_fileutils.h" // _Py_stat_struct
29 #include "structmember.h" // PyMemberDef
30 #include <stddef.h> // offsetof()
31
32 #ifndef MS_WINDOWS
33 #define UNIX
34 # ifdef HAVE_FCNTL_H
35 # include <fcntl.h>
36 # endif /* HAVE_FCNTL_H */
37 #endif
38
39 #ifdef MS_WINDOWS
40 #include <windows.h>
41 static int
my_getpagesize(void)42 my_getpagesize(void)
43 {
44 SYSTEM_INFO si;
45 GetSystemInfo(&si);
46 return si.dwPageSize;
47 }
48
49 static int
my_getallocationgranularity(void)50 my_getallocationgranularity (void)
51 {
52
53 SYSTEM_INFO si;
54 GetSystemInfo(&si);
55 return si.dwAllocationGranularity;
56 }
57
58 #endif
59
60 #ifdef UNIX
61 #include <sys/mman.h>
62 #include <sys/stat.h>
63
64 #if defined(HAVE_SYSCONF) && defined(_SC_PAGESIZE)
65 static int
my_getpagesize(void)66 my_getpagesize(void)
67 {
68 return sysconf(_SC_PAGESIZE);
69 }
70
71 #define my_getallocationgranularity my_getpagesize
72 #else
73 #define my_getpagesize getpagesize
74 #endif
75
76 #endif /* UNIX */
77
78 #include <string.h>
79
80 #ifdef HAVE_SYS_TYPES_H
81 #include <sys/types.h>
82 #endif /* HAVE_SYS_TYPES_H */
83
84 /* Prefer MAP_ANONYMOUS since MAP_ANON is deprecated according to man page. */
85 #if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
86 # define MAP_ANONYMOUS MAP_ANON
87 #endif
88
89 typedef enum
90 {
91 ACCESS_DEFAULT,
92 ACCESS_READ,
93 ACCESS_WRITE,
94 ACCESS_COPY
95 } access_mode;
96
97 typedef struct {
98 PyObject_HEAD
99 char * data;
100 Py_ssize_t size;
101 Py_ssize_t pos; /* relative to offset */
102 #ifdef MS_WINDOWS
103 long long offset;
104 #else
105 off_t offset;
106 #endif
107 Py_ssize_t exports;
108
109 #ifdef MS_WINDOWS
110 HANDLE map_handle;
111 HANDLE file_handle;
112 char * tagname;
113 #endif
114
115 #ifdef UNIX
116 int fd;
117 #endif
118
119 PyObject *weakreflist;
120 access_mode access;
121 } mmap_object;
122
123 typedef struct {
124 PyTypeObject *mmap_object_type;
125 } mmap_state;
126
127 static mmap_state *
get_mmap_state(PyObject * module)128 get_mmap_state(PyObject *module)
129 {
130 mmap_state *state = PyModule_GetState(module);
131 assert(state);
132 return state;
133 }
134
135 static int
mmap_object_traverse(mmap_object * m_obj,visitproc visit,void * arg)136 mmap_object_traverse(mmap_object *m_obj, visitproc visit, void *arg)
137 {
138 Py_VISIT(Py_TYPE(m_obj));
139 return 0;
140 }
141
142 static void
mmap_object_dealloc(mmap_object * m_obj)143 mmap_object_dealloc(mmap_object *m_obj)
144 {
145 PyTypeObject *tp = Py_TYPE(m_obj);
146 PyObject_GC_UnTrack(m_obj);
147
148 #ifdef MS_WINDOWS
149 Py_BEGIN_ALLOW_THREADS
150 if (m_obj->data != NULL)
151 UnmapViewOfFile (m_obj->data);
152 if (m_obj->map_handle != NULL)
153 CloseHandle (m_obj->map_handle);
154 if (m_obj->file_handle != INVALID_HANDLE_VALUE)
155 CloseHandle (m_obj->file_handle);
156 Py_END_ALLOW_THREADS
157 if (m_obj->tagname)
158 PyMem_Free(m_obj->tagname);
159 #endif /* MS_WINDOWS */
160
161 #ifdef UNIX
162 Py_BEGIN_ALLOW_THREADS
163 if (m_obj->fd >= 0)
164 (void) close(m_obj->fd);
165 if (m_obj->data!=NULL) {
166 munmap(m_obj->data, m_obj->size);
167 }
168 Py_END_ALLOW_THREADS
169 #endif /* UNIX */
170
171 if (m_obj->weakreflist != NULL)
172 PyObject_ClearWeakRefs((PyObject *) m_obj);
173
174 tp->tp_free(m_obj);
175 Py_DECREF(tp);
176 }
177
178 static PyObject *
mmap_close_method(mmap_object * self,PyObject * unused)179 mmap_close_method(mmap_object *self, PyObject *unused)
180 {
181 if (self->exports > 0) {
182 PyErr_SetString(PyExc_BufferError, "cannot close "\
183 "exported pointers exist");
184 return NULL;
185 }
186 #ifdef MS_WINDOWS
187 /* For each resource we maintain, we need to check
188 the value is valid, and if so, free the resource
189 and set the member value to an invalid value so
190 the dealloc does not attempt to resource clearing
191 again.
192 TODO - should we check for errors in the close operations???
193 */
194 HANDLE map_handle = self->map_handle;
195 HANDLE file_handle = self->file_handle;
196 char *data = self->data;
197 self->map_handle = NULL;
198 self->file_handle = INVALID_HANDLE_VALUE;
199 self->data = NULL;
200 Py_BEGIN_ALLOW_THREADS
201 if (data != NULL) {
202 UnmapViewOfFile(data);
203 }
204 if (map_handle != NULL) {
205 CloseHandle(map_handle);
206 }
207 if (file_handle != INVALID_HANDLE_VALUE) {
208 CloseHandle(file_handle);
209 }
210 Py_END_ALLOW_THREADS
211 #endif /* MS_WINDOWS */
212
213 #ifdef UNIX
214 int fd = self->fd;
215 char *data = self->data;
216 self->fd = -1;
217 self->data = NULL;
218 Py_BEGIN_ALLOW_THREADS
219 if (0 <= fd)
220 (void) close(fd);
221 if (data != NULL) {
222 munmap(data, self->size);
223 }
224 Py_END_ALLOW_THREADS
225 #endif
226
227 Py_RETURN_NONE;
228 }
229
230 #ifdef MS_WINDOWS
231 #define CHECK_VALID(err) \
232 do { \
233 if (self->map_handle == NULL) { \
234 PyErr_SetString(PyExc_ValueError, "mmap closed or invalid"); \
235 return err; \
236 } \
237 } while (0)
238 #define CHECK_VALID_OR_RELEASE(err, buffer) \
239 do { \
240 if (self->map_handle == NULL) { \
241 PyErr_SetString(PyExc_ValueError, "mmap closed or invalid"); \
242 PyBuffer_Release(&(buffer)); \
243 return (err); \
244 } \
245 } while (0)
246 #endif /* MS_WINDOWS */
247
248 #ifdef UNIX
249 #define CHECK_VALID(err) \
250 do { \
251 if (self->data == NULL) { \
252 PyErr_SetString(PyExc_ValueError, "mmap closed or invalid"); \
253 return err; \
254 } \
255 } while (0)
256 #define CHECK_VALID_OR_RELEASE(err, buffer) \
257 do { \
258 if (self->data == NULL) { \
259 PyErr_SetString(PyExc_ValueError, "mmap closed or invalid"); \
260 PyBuffer_Release(&(buffer)); \
261 return (err); \
262 } \
263 } while (0)
264 #endif /* UNIX */
265
266 static PyObject *
mmap_read_byte_method(mmap_object * self,PyObject * unused)267 mmap_read_byte_method(mmap_object *self,
268 PyObject *unused)
269 {
270 CHECK_VALID(NULL);
271 if (self->pos >= self->size) {
272 PyErr_SetString(PyExc_ValueError, "read byte out of range");
273 return NULL;
274 }
275 return PyLong_FromLong((unsigned char)self->data[self->pos++]);
276 }
277
278 static PyObject *
mmap_read_line_method(mmap_object * self,PyObject * unused)279 mmap_read_line_method(mmap_object *self,
280 PyObject *unused)
281 {
282 Py_ssize_t remaining;
283 char *start, *eol;
284 PyObject *result;
285
286 CHECK_VALID(NULL);
287
288 remaining = (self->pos < self->size) ? self->size - self->pos : 0;
289 if (!remaining)
290 return PyBytes_FromString("");
291 start = self->data + self->pos;
292 eol = memchr(start, '\n', remaining);
293 if (!eol)
294 eol = self->data + self->size;
295 else
296 ++eol; /* advance past newline */
297 result = PyBytes_FromStringAndSize(start, (eol - start));
298 self->pos += (eol - start);
299 return result;
300 }
301
302 static PyObject *
mmap_read_method(mmap_object * self,PyObject * args)303 mmap_read_method(mmap_object *self,
304 PyObject *args)
305 {
306 Py_ssize_t num_bytes = PY_SSIZE_T_MAX, remaining;
307 PyObject *result;
308
309 CHECK_VALID(NULL);
310 if (!PyArg_ParseTuple(args, "|O&:read", _Py_convert_optional_to_ssize_t, &num_bytes))
311 return NULL;
312 CHECK_VALID(NULL);
313
314 /* silently 'adjust' out-of-range requests */
315 remaining = (self->pos < self->size) ? self->size - self->pos : 0;
316 if (num_bytes < 0 || num_bytes > remaining)
317 num_bytes = remaining;
318 result = PyBytes_FromStringAndSize(&self->data[self->pos], num_bytes);
319 self->pos += num_bytes;
320 return result;
321 }
322
323 static PyObject *
mmap_gfind(mmap_object * self,PyObject * args,int reverse)324 mmap_gfind(mmap_object *self,
325 PyObject *args,
326 int reverse)
327 {
328 Py_ssize_t start = self->pos;
329 Py_ssize_t end = self->size;
330 Py_buffer view;
331
332 CHECK_VALID(NULL);
333 if (!PyArg_ParseTuple(args, reverse ? "y*|nn:rfind" : "y*|nn:find",
334 &view, &start, &end)) {
335 return NULL;
336 }
337 else {
338 if (start < 0)
339 start += self->size;
340 if (start < 0)
341 start = 0;
342 else if (start > self->size)
343 start = self->size;
344
345 if (end < 0)
346 end += self->size;
347 if (end < 0)
348 end = 0;
349 else if (end > self->size)
350 end = self->size;
351
352 Py_ssize_t res;
353 CHECK_VALID_OR_RELEASE(NULL, view);
354 if (reverse) {
355 res = _PyBytes_ReverseFind(
356 self->data + start, end - start,
357 view.buf, view.len, start);
358 }
359 else {
360 res = _PyBytes_Find(
361 self->data + start, end - start,
362 view.buf, view.len, start);
363 }
364 PyBuffer_Release(&view);
365 return PyLong_FromSsize_t(res);
366 }
367 }
368
369 static PyObject *
mmap_find_method(mmap_object * self,PyObject * args)370 mmap_find_method(mmap_object *self,
371 PyObject *args)
372 {
373 return mmap_gfind(self, args, 0);
374 }
375
376 static PyObject *
mmap_rfind_method(mmap_object * self,PyObject * args)377 mmap_rfind_method(mmap_object *self,
378 PyObject *args)
379 {
380 return mmap_gfind(self, args, 1);
381 }
382
383 static int
is_writable(mmap_object * self)384 is_writable(mmap_object *self)
385 {
386 if (self->access != ACCESS_READ)
387 return 1;
388 PyErr_Format(PyExc_TypeError, "mmap can't modify a readonly memory map.");
389 return 0;
390 }
391
392 static int
is_resizeable(mmap_object * self)393 is_resizeable(mmap_object *self)
394 {
395 if (self->exports > 0) {
396 PyErr_SetString(PyExc_BufferError,
397 "mmap can't resize with extant buffers exported.");
398 return 0;
399 }
400 if ((self->access == ACCESS_WRITE) || (self->access == ACCESS_DEFAULT))
401 return 1;
402 PyErr_Format(PyExc_TypeError,
403 "mmap can't resize a readonly or copy-on-write memory map.");
404 return 0;
405
406 }
407
408
409 static PyObject *
mmap_write_method(mmap_object * self,PyObject * args)410 mmap_write_method(mmap_object *self,
411 PyObject *args)
412 {
413 Py_buffer data;
414
415 CHECK_VALID(NULL);
416 if (!PyArg_ParseTuple(args, "y*:write", &data))
417 return NULL;
418
419 if (!is_writable(self)) {
420 PyBuffer_Release(&data);
421 return NULL;
422 }
423
424 if (self->pos > self->size || self->size - self->pos < data.len) {
425 PyBuffer_Release(&data);
426 PyErr_SetString(PyExc_ValueError, "data out of range");
427 return NULL;
428 }
429
430 CHECK_VALID_OR_RELEASE(NULL, data);
431 memcpy(&self->data[self->pos], data.buf, data.len);
432 self->pos += data.len;
433 PyBuffer_Release(&data);
434 return PyLong_FromSsize_t(data.len);
435 }
436
437 static PyObject *
mmap_write_byte_method(mmap_object * self,PyObject * args)438 mmap_write_byte_method(mmap_object *self,
439 PyObject *args)
440 {
441 char value;
442
443 CHECK_VALID(NULL);
444 if (!PyArg_ParseTuple(args, "b:write_byte", &value))
445 return(NULL);
446
447 if (!is_writable(self))
448 return NULL;
449
450 CHECK_VALID(NULL);
451 if (self->pos < self->size) {
452 self->data[self->pos++] = value;
453 Py_RETURN_NONE;
454 }
455 else {
456 PyErr_SetString(PyExc_ValueError, "write byte out of range");
457 return NULL;
458 }
459 }
460
461 static PyObject *
mmap_size_method(mmap_object * self,PyObject * unused)462 mmap_size_method(mmap_object *self,
463 PyObject *unused)
464 {
465 CHECK_VALID(NULL);
466
467 #ifdef MS_WINDOWS
468 if (self->file_handle != INVALID_HANDLE_VALUE) {
469 DWORD low,high;
470 long long size;
471 low = GetFileSize(self->file_handle, &high);
472 if (low == INVALID_FILE_SIZE) {
473 /* It might be that the function appears to have failed,
474 when indeed its size equals INVALID_FILE_SIZE */
475 DWORD error = GetLastError();
476 if (error != NO_ERROR)
477 return PyErr_SetFromWindowsErr(error);
478 }
479 if (!high && low < LONG_MAX)
480 return PyLong_FromLong((long)low);
481 size = (((long long)high)<<32) + low;
482 return PyLong_FromLongLong(size);
483 } else {
484 return PyLong_FromSsize_t(self->size);
485 }
486 #endif /* MS_WINDOWS */
487
488 #ifdef UNIX
489 {
490 struct _Py_stat_struct status;
491 if (_Py_fstat(self->fd, &status) == -1)
492 return NULL;
493 #ifdef HAVE_LARGEFILE_SUPPORT
494 return PyLong_FromLongLong(status.st_size);
495 #else
496 return PyLong_FromLong(status.st_size);
497 #endif
498 }
499 #endif /* UNIX */
500 }
501
502 /* This assumes that you want the entire file mapped,
503 / and when recreating the map will make the new file
504 / have the new size
505 /
506 / Is this really necessary? This could easily be done
507 / from python by just closing and re-opening with the
508 / new size?
509 */
510
511 static PyObject *
mmap_resize_method(mmap_object * self,PyObject * args)512 mmap_resize_method(mmap_object *self,
513 PyObject *args)
514 {
515 Py_ssize_t new_size;
516 CHECK_VALID(NULL);
517 if (!PyArg_ParseTuple(args, "n:resize", &new_size) ||
518 !is_resizeable(self)) {
519 return NULL;
520 }
521 if (new_size < 0 || PY_SSIZE_T_MAX - new_size < self->offset) {
522 PyErr_SetString(PyExc_ValueError, "new size out of range");
523 return NULL;
524 }
525
526 {
527 #ifdef MS_WINDOWS
528 DWORD error = 0, file_resize_error = 0;
529 char* old_data = self->data;
530 LARGE_INTEGER offset, max_size;
531 offset.QuadPart = self->offset;
532 max_size.QuadPart = self->offset + new_size;
533 /* close the file mapping */
534 CloseHandle(self->map_handle);
535 /* if the file mapping still exists, it cannot be resized. */
536 if (self->tagname) {
537 self->map_handle = OpenFileMapping(FILE_MAP_WRITE, FALSE,
538 self->tagname);
539 if (self->map_handle) {
540 PyErr_SetFromWindowsErr(ERROR_USER_MAPPED_FILE);
541 return NULL;
542 }
543 } else {
544 self->map_handle = NULL;
545 }
546
547 /* if it's not the paging file, unmap the view and resize the file */
548 if (self->file_handle != INVALID_HANDLE_VALUE) {
549 if (!UnmapViewOfFile(self->data)) {
550 return PyErr_SetFromWindowsErr(GetLastError());
551 };
552 self->data = NULL;
553 /* resize the file */
554 if (!SetFilePointerEx(self->file_handle, max_size, NULL,
555 FILE_BEGIN) ||
556 !SetEndOfFile(self->file_handle)) {
557 /* resizing failed. try to remap the file */
558 file_resize_error = GetLastError();
559 max_size.QuadPart = self->size;
560 new_size = self->size;
561 }
562 }
563
564 /* create a new file mapping and map a new view */
565 /* FIXME: call CreateFileMappingW with wchar_t tagname */
566 self->map_handle = CreateFileMapping(
567 self->file_handle,
568 NULL,
569 PAGE_READWRITE,
570 max_size.HighPart,
571 max_size.LowPart,
572 self->tagname);
573
574 error = GetLastError();
575 /* ERROR_ALREADY_EXISTS implies that between our closing the handle above and
576 calling CreateFileMapping here, someone's created a different mapping with
577 the same name. There's nothing we can usefully do so we invalidate our
578 mapping and error out.
579 */
580 if (error == ERROR_ALREADY_EXISTS) {
581 CloseHandle(self->map_handle);
582 self->map_handle = NULL;
583 }
584 else if (self->map_handle != NULL) {
585 self->data = MapViewOfFile(self->map_handle,
586 FILE_MAP_WRITE,
587 offset.HighPart,
588 offset.LowPart,
589 new_size);
590 if (self->data != NULL) {
591 /* copy the old view if using the paging file */
592 if (self->file_handle == INVALID_HANDLE_VALUE) {
593 memcpy(self->data, old_data,
594 self->size < new_size ? self->size : new_size);
595 if (!UnmapViewOfFile(old_data)) {
596 error = GetLastError();
597 }
598 }
599 self->size = new_size;
600 }
601 else {
602 error = GetLastError();
603 CloseHandle(self->map_handle);
604 self->map_handle = NULL;
605 }
606 }
607
608 if (error) {
609 return PyErr_SetFromWindowsErr(error);
610 return NULL;
611 }
612 /* It's possible for a resize to fail, typically because another mapping
613 is still held against the same underlying file. Even if nothing has
614 failed -- ie we're still returning a valid file mapping -- raise the
615 error as an exception as the resize won't have happened
616 */
617 if (file_resize_error) {
618 PyErr_SetFromWindowsErr(file_resize_error);
619 return NULL;
620 }
621 Py_RETURN_NONE;
622 #endif /* MS_WINDOWS */
623
624 #ifdef UNIX
625 #ifndef HAVE_MREMAP
626 PyErr_SetString(PyExc_SystemError,
627 "mmap: resizing not available--no mremap()");
628 return NULL;
629 #else
630 void *newmap;
631
632 if (self->fd != -1 && ftruncate(self->fd, self->offset + new_size) == -1) {
633 PyErr_SetFromErrno(PyExc_OSError);
634 return NULL;
635 }
636
637 #ifdef MREMAP_MAYMOVE
638 newmap = mremap(self->data, self->size, new_size, MREMAP_MAYMOVE);
639 #else
640 #if defined(__NetBSD__)
641 newmap = mremap(self->data, self->size, self->data, new_size, 0);
642 #else
643 newmap = mremap(self->data, self->size, new_size, 0);
644 #endif /* __NetBSD__ */
645 #endif
646 if (newmap == (void *)-1)
647 {
648 PyErr_SetFromErrno(PyExc_OSError);
649 return NULL;
650 }
651 self->data = newmap;
652 self->size = new_size;
653 Py_RETURN_NONE;
654 #endif /* HAVE_MREMAP */
655 #endif /* UNIX */
656 }
657 }
658
659 static PyObject *
mmap_tell_method(mmap_object * self,PyObject * unused)660 mmap_tell_method(mmap_object *self, PyObject *unused)
661 {
662 CHECK_VALID(NULL);
663 return PyLong_FromSize_t(self->pos);
664 }
665
666 static PyObject *
mmap_flush_method(mmap_object * self,PyObject * args)667 mmap_flush_method(mmap_object *self, PyObject *args)
668 {
669 Py_ssize_t offset = 0;
670 Py_ssize_t size = self->size;
671 CHECK_VALID(NULL);
672 if (!PyArg_ParseTuple(args, "|nn:flush", &offset, &size))
673 return NULL;
674 if (size < 0 || offset < 0 || self->size - offset < size) {
675 PyErr_SetString(PyExc_ValueError, "flush values out of range");
676 return NULL;
677 }
678
679 if (self->access == ACCESS_READ || self->access == ACCESS_COPY)
680 Py_RETURN_NONE;
681
682 #ifdef MS_WINDOWS
683 if (!FlushViewOfFile(self->data+offset, size)) {
684 PyErr_SetFromWindowsErr(GetLastError());
685 return NULL;
686 }
687 Py_RETURN_NONE;
688 #elif defined(UNIX)
689 /* XXX flags for msync? */
690 if (-1 == msync(self->data + offset, size, MS_SYNC)) {
691 PyErr_SetFromErrno(PyExc_OSError);
692 return NULL;
693 }
694 Py_RETURN_NONE;
695 #else
696 PyErr_SetString(PyExc_ValueError, "flush not supported on this system");
697 return NULL;
698 #endif
699 }
700
701 static PyObject *
mmap_seek_method(mmap_object * self,PyObject * args)702 mmap_seek_method(mmap_object *self, PyObject *args)
703 {
704 Py_ssize_t dist;
705 int how=0;
706 CHECK_VALID(NULL);
707 if (!PyArg_ParseTuple(args, "n|i:seek", &dist, &how))
708 return NULL;
709 else {
710 Py_ssize_t where;
711 switch (how) {
712 case 0: /* relative to start */
713 where = dist;
714 break;
715 case 1: /* relative to current position */
716 if (PY_SSIZE_T_MAX - self->pos < dist)
717 goto onoutofrange;
718 where = self->pos + dist;
719 break;
720 case 2: /* relative to end */
721 if (PY_SSIZE_T_MAX - self->size < dist)
722 goto onoutofrange;
723 where = self->size + dist;
724 break;
725 default:
726 PyErr_SetString(PyExc_ValueError, "unknown seek type");
727 return NULL;
728 }
729 if (where > self->size || where < 0)
730 goto onoutofrange;
731 self->pos = where;
732 Py_RETURN_NONE;
733 }
734
735 onoutofrange:
736 PyErr_SetString(PyExc_ValueError, "seek out of range");
737 return NULL;
738 }
739
740 static PyObject *
mmap_move_method(mmap_object * self,PyObject * args)741 mmap_move_method(mmap_object *self, PyObject *args)
742 {
743 Py_ssize_t dest, src, cnt;
744 CHECK_VALID(NULL);
745 if (!PyArg_ParseTuple(args, "nnn:move", &dest, &src, &cnt) ||
746 !is_writable(self)) {
747 return NULL;
748 } else {
749 /* bounds check the values */
750 if (dest < 0 || src < 0 || cnt < 0)
751 goto bounds;
752 if (self->size - dest < cnt || self->size - src < cnt)
753 goto bounds;
754
755 CHECK_VALID(NULL);
756 memmove(&self->data[dest], &self->data[src], cnt);
757
758 Py_RETURN_NONE;
759
760 bounds:
761 PyErr_SetString(PyExc_ValueError,
762 "source, destination, or count out of range");
763 return NULL;
764 }
765 }
766
767 static PyObject *
mmap_closed_get(mmap_object * self,void * Py_UNUSED (ignored))768 mmap_closed_get(mmap_object *self, void *Py_UNUSED(ignored))
769 {
770 #ifdef MS_WINDOWS
771 return PyBool_FromLong(self->map_handle == NULL ? 1 : 0);
772 #elif defined(UNIX)
773 return PyBool_FromLong(self->data == NULL ? 1 : 0);
774 #endif
775 }
776
777 static PyObject *
mmap__enter__method(mmap_object * self,PyObject * args)778 mmap__enter__method(mmap_object *self, PyObject *args)
779 {
780 CHECK_VALID(NULL);
781
782 Py_INCREF(self);
783 return (PyObject *)self;
784 }
785
786 static PyObject *
mmap__exit__method(PyObject * self,PyObject * args)787 mmap__exit__method(PyObject *self, PyObject *args)
788 {
789 return mmap_close_method((mmap_object *)self, NULL);
790 }
791
792 static PyObject *
mmap__repr__method(PyObject * self)793 mmap__repr__method(PyObject *self)
794 {
795 mmap_object *mobj = (mmap_object *)self;
796
797 #ifdef MS_WINDOWS
798 #define _Py_FORMAT_OFFSET "lld"
799 if (mobj->map_handle == NULL)
800 #elif defined(UNIX)
801 # ifdef HAVE_LARGEFILE_SUPPORT
802 # define _Py_FORMAT_OFFSET "lld"
803 # else
804 # define _Py_FORMAT_OFFSET "ld"
805 # endif
806 if (mobj->data == NULL)
807 #endif
808 {
809 return PyUnicode_FromFormat("<%s closed=True>", Py_TYPE(self)->tp_name);
810 } else {
811 const char *access_str;
812
813 switch (mobj->access) {
814 case ACCESS_DEFAULT:
815 access_str = "ACCESS_DEFAULT";
816 break;
817 case ACCESS_READ:
818 access_str = "ACCESS_READ";
819 break;
820 case ACCESS_WRITE:
821 access_str = "ACCESS_WRITE";
822 break;
823 case ACCESS_COPY:
824 access_str = "ACCESS_COPY";
825 break;
826 default:
827 Py_UNREACHABLE();
828 }
829
830 return PyUnicode_FromFormat("<%s closed=False, access=%s, length=%zd, "
831 "pos=%zd, offset=%" _Py_FORMAT_OFFSET ">",
832 Py_TYPE(self)->tp_name, access_str,
833 mobj->size, mobj->pos, mobj->offset);
834 }
835 }
836
837 #ifdef MS_WINDOWS
838 static PyObject *
mmap__sizeof__method(mmap_object * self,void * unused)839 mmap__sizeof__method(mmap_object *self, void *unused)
840 {
841 Py_ssize_t res;
842
843 res = _PyObject_SIZE(Py_TYPE(self));
844 if (self->tagname)
845 res += strlen(self->tagname) + 1;
846 return PyLong_FromSsize_t(res);
847 }
848 #endif
849
850 #ifdef HAVE_MADVISE
851 static PyObject *
mmap_madvise_method(mmap_object * self,PyObject * args)852 mmap_madvise_method(mmap_object *self, PyObject *args)
853 {
854 int option;
855 Py_ssize_t start = 0, length;
856
857 CHECK_VALID(NULL);
858 length = self->size;
859
860 if (!PyArg_ParseTuple(args, "i|nn:madvise", &option, &start, &length)) {
861 return NULL;
862 }
863
864 if (start < 0 || start >= self->size) {
865 PyErr_SetString(PyExc_ValueError, "madvise start out of bounds");
866 return NULL;
867 }
868 if (length < 0) {
869 PyErr_SetString(PyExc_ValueError, "madvise length invalid");
870 return NULL;
871 }
872 if (PY_SSIZE_T_MAX - start < length) {
873 PyErr_SetString(PyExc_OverflowError, "madvise length too large");
874 return NULL;
875 }
876
877 if (start + length > self->size) {
878 length = self->size - start;
879 }
880
881 CHECK_VALID(NULL);
882 if (madvise(self->data + start, length, option) != 0) {
883 PyErr_SetFromErrno(PyExc_OSError);
884 return NULL;
885 }
886
887 Py_RETURN_NONE;
888 }
889 #endif // HAVE_MADVISE
890
891 static struct PyMemberDef mmap_object_members[] = {
892 {"__weaklistoffset__", T_PYSSIZET, offsetof(mmap_object, weakreflist), READONLY},
893 {NULL},
894 };
895
896 static struct PyMethodDef mmap_object_methods[] = {
897 {"close", (PyCFunction) mmap_close_method, METH_NOARGS},
898 {"find", (PyCFunction) mmap_find_method, METH_VARARGS},
899 {"rfind", (PyCFunction) mmap_rfind_method, METH_VARARGS},
900 {"flush", (PyCFunction) mmap_flush_method, METH_VARARGS},
901 #ifdef HAVE_MADVISE
902 {"madvise", (PyCFunction) mmap_madvise_method, METH_VARARGS},
903 #endif
904 {"move", (PyCFunction) mmap_move_method, METH_VARARGS},
905 {"read", (PyCFunction) mmap_read_method, METH_VARARGS},
906 {"read_byte", (PyCFunction) mmap_read_byte_method, METH_NOARGS},
907 {"readline", (PyCFunction) mmap_read_line_method, METH_NOARGS},
908 {"resize", (PyCFunction) mmap_resize_method, METH_VARARGS},
909 {"seek", (PyCFunction) mmap_seek_method, METH_VARARGS},
910 {"size", (PyCFunction) mmap_size_method, METH_NOARGS},
911 {"tell", (PyCFunction) mmap_tell_method, METH_NOARGS},
912 {"write", (PyCFunction) mmap_write_method, METH_VARARGS},
913 {"write_byte", (PyCFunction) mmap_write_byte_method, METH_VARARGS},
914 {"__enter__", (PyCFunction) mmap__enter__method, METH_NOARGS},
915 {"__exit__", (PyCFunction) mmap__exit__method, METH_VARARGS},
916 #ifdef MS_WINDOWS
917 {"__sizeof__", (PyCFunction) mmap__sizeof__method, METH_NOARGS},
918 #endif
919 {NULL, NULL} /* sentinel */
920 };
921
922 static PyGetSetDef mmap_object_getset[] = {
923 {"closed", (getter) mmap_closed_get, NULL, NULL},
924 {NULL}
925 };
926
927
928 /* Functions for treating an mmap'ed file as a buffer */
929
930 static int
mmap_buffer_getbuf(mmap_object * self,Py_buffer * view,int flags)931 mmap_buffer_getbuf(mmap_object *self, Py_buffer *view, int flags)
932 {
933 CHECK_VALID(-1);
934 if (PyBuffer_FillInfo(view, (PyObject*)self, self->data, self->size,
935 (self->access == ACCESS_READ), flags) < 0)
936 return -1;
937 self->exports++;
938 return 0;
939 }
940
941 static void
mmap_buffer_releasebuf(mmap_object * self,Py_buffer * view)942 mmap_buffer_releasebuf(mmap_object *self, Py_buffer *view)
943 {
944 self->exports--;
945 }
946
947 static Py_ssize_t
mmap_length(mmap_object * self)948 mmap_length(mmap_object *self)
949 {
950 CHECK_VALID(-1);
951 return self->size;
952 }
953
954 static PyObject *
mmap_item(mmap_object * self,Py_ssize_t i)955 mmap_item(mmap_object *self, Py_ssize_t i)
956 {
957 CHECK_VALID(NULL);
958 if (i < 0 || i >= self->size) {
959 PyErr_SetString(PyExc_IndexError, "mmap index out of range");
960 return NULL;
961 }
962 return PyBytes_FromStringAndSize(self->data + i, 1);
963 }
964
965 static PyObject *
mmap_subscript(mmap_object * self,PyObject * item)966 mmap_subscript(mmap_object *self, PyObject *item)
967 {
968 CHECK_VALID(NULL);
969 if (PyIndex_Check(item)) {
970 Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError);
971 if (i == -1 && PyErr_Occurred())
972 return NULL;
973 if (i < 0)
974 i += self->size;
975 if (i < 0 || i >= self->size) {
976 PyErr_SetString(PyExc_IndexError,
977 "mmap index out of range");
978 return NULL;
979 }
980 CHECK_VALID(NULL);
981 return PyLong_FromLong(Py_CHARMASK(self->data[i]));
982 }
983 else if (PySlice_Check(item)) {
984 Py_ssize_t start, stop, step, slicelen;
985
986 if (PySlice_Unpack(item, &start, &stop, &step) < 0) {
987 return NULL;
988 }
989 slicelen = PySlice_AdjustIndices(self->size, &start, &stop, step);
990
991 CHECK_VALID(NULL);
992 if (slicelen <= 0)
993 return PyBytes_FromStringAndSize("", 0);
994 else if (step == 1)
995 return PyBytes_FromStringAndSize(self->data + start,
996 slicelen);
997 else {
998 char *result_buf = (char *)PyMem_Malloc(slicelen);
999 size_t cur;
1000 Py_ssize_t i;
1001 PyObject *result;
1002
1003 if (result_buf == NULL)
1004 return PyErr_NoMemory();
1005
1006 for (cur = start, i = 0; i < slicelen;
1007 cur += step, i++) {
1008 result_buf[i] = self->data[cur];
1009 }
1010 result = PyBytes_FromStringAndSize(result_buf,
1011 slicelen);
1012 PyMem_Free(result_buf);
1013 return result;
1014 }
1015 }
1016 else {
1017 PyErr_SetString(PyExc_TypeError,
1018 "mmap indices must be integers");
1019 return NULL;
1020 }
1021 }
1022
1023 static int
mmap_ass_item(mmap_object * self,Py_ssize_t i,PyObject * v)1024 mmap_ass_item(mmap_object *self, Py_ssize_t i, PyObject *v)
1025 {
1026 const char *buf;
1027
1028 CHECK_VALID(-1);
1029 if (i < 0 || i >= self->size) {
1030 PyErr_SetString(PyExc_IndexError, "mmap index out of range");
1031 return -1;
1032 }
1033 if (v == NULL) {
1034 PyErr_SetString(PyExc_TypeError,
1035 "mmap object doesn't support item deletion");
1036 return -1;
1037 }
1038 if (! (PyBytes_Check(v) && PyBytes_Size(v)==1) ) {
1039 PyErr_SetString(PyExc_IndexError,
1040 "mmap assignment must be length-1 bytes()");
1041 return -1;
1042 }
1043 if (!is_writable(self))
1044 return -1;
1045 buf = PyBytes_AsString(v);
1046 self->data[i] = buf[0];
1047 return 0;
1048 }
1049
1050 static int
mmap_ass_subscript(mmap_object * self,PyObject * item,PyObject * value)1051 mmap_ass_subscript(mmap_object *self, PyObject *item, PyObject *value)
1052 {
1053 CHECK_VALID(-1);
1054
1055 if (!is_writable(self))
1056 return -1;
1057
1058 if (PyIndex_Check(item)) {
1059 Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError);
1060 Py_ssize_t v;
1061
1062 if (i == -1 && PyErr_Occurred())
1063 return -1;
1064 if (i < 0)
1065 i += self->size;
1066 if (i < 0 || i >= self->size) {
1067 PyErr_SetString(PyExc_IndexError,
1068 "mmap index out of range");
1069 return -1;
1070 }
1071 if (value == NULL) {
1072 PyErr_SetString(PyExc_TypeError,
1073 "mmap doesn't support item deletion");
1074 return -1;
1075 }
1076 if (!PyIndex_Check(value)) {
1077 PyErr_SetString(PyExc_TypeError,
1078 "mmap item value must be an int");
1079 return -1;
1080 }
1081 v = PyNumber_AsSsize_t(value, PyExc_TypeError);
1082 if (v == -1 && PyErr_Occurred())
1083 return -1;
1084 if (v < 0 || v > 255) {
1085 PyErr_SetString(PyExc_ValueError,
1086 "mmap item value must be "
1087 "in range(0, 256)");
1088 return -1;
1089 }
1090 CHECK_VALID(-1);
1091 self->data[i] = (char) v;
1092 return 0;
1093 }
1094 else if (PySlice_Check(item)) {
1095 Py_ssize_t start, stop, step, slicelen;
1096 Py_buffer vbuf;
1097
1098 if (PySlice_Unpack(item, &start, &stop, &step) < 0) {
1099 return -1;
1100 }
1101 slicelen = PySlice_AdjustIndices(self->size, &start, &stop, step);
1102 if (value == NULL) {
1103 PyErr_SetString(PyExc_TypeError,
1104 "mmap object doesn't support slice deletion");
1105 return -1;
1106 }
1107 if (PyObject_GetBuffer(value, &vbuf, PyBUF_SIMPLE) < 0)
1108 return -1;
1109 if (vbuf.len != slicelen) {
1110 PyErr_SetString(PyExc_IndexError,
1111 "mmap slice assignment is wrong size");
1112 PyBuffer_Release(&vbuf);
1113 return -1;
1114 }
1115
1116 CHECK_VALID_OR_RELEASE(-1, vbuf);
1117 if (slicelen == 0) {
1118 }
1119 else if (step == 1) {
1120 memcpy(self->data + start, vbuf.buf, slicelen);
1121 }
1122 else {
1123 size_t cur;
1124 Py_ssize_t i;
1125
1126 for (cur = start, i = 0;
1127 i < slicelen;
1128 cur += step, i++)
1129 {
1130 self->data[cur] = ((char *)vbuf.buf)[i];
1131 }
1132 }
1133 PyBuffer_Release(&vbuf);
1134 return 0;
1135 }
1136 else {
1137 PyErr_SetString(PyExc_TypeError,
1138 "mmap indices must be integer");
1139 return -1;
1140 }
1141 }
1142
1143 static PyObject *
1144 new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict);
1145
1146 PyDoc_STRVAR(mmap_doc,
1147 "Windows: mmap(fileno, length[, tagname[, access[, offset]]])\n\
1148 \n\
1149 Maps length bytes from the file specified by the file handle fileno,\n\
1150 and returns a mmap object. If length is larger than the current size\n\
1151 of the file, the file is extended to contain length bytes. If length\n\
1152 is 0, the maximum length of the map is the current size of the file,\n\
1153 except that if the file is empty Windows raises an exception (you cannot\n\
1154 create an empty mapping on Windows).\n\
1155 \n\
1156 Unix: mmap(fileno, length[, flags[, prot[, access[, offset]]]])\n\
1157 \n\
1158 Maps length bytes from the file specified by the file descriptor fileno,\n\
1159 and returns a mmap object. If length is 0, the maximum length of the map\n\
1160 will be the current size of the file when mmap is called.\n\
1161 flags specifies the nature of the mapping. MAP_PRIVATE creates a\n\
1162 private copy-on-write mapping, so changes to the contents of the mmap\n\
1163 object will be private to this process, and MAP_SHARED creates a mapping\n\
1164 that's shared with all other processes mapping the same areas of the file.\n\
1165 The default value is MAP_SHARED.\n\
1166 \n\
1167 To map anonymous memory, pass -1 as the fileno (both versions).");
1168
1169
1170 static PyType_Slot mmap_object_slots[] = {
1171 {Py_tp_new, new_mmap_object},
1172 {Py_tp_dealloc, mmap_object_dealloc},
1173 {Py_tp_repr, mmap__repr__method},
1174 {Py_tp_doc, (void *)mmap_doc},
1175 {Py_tp_methods, mmap_object_methods},
1176 {Py_tp_members, mmap_object_members},
1177 {Py_tp_getset, mmap_object_getset},
1178 {Py_tp_getattro, PyObject_GenericGetAttr},
1179 {Py_tp_traverse, mmap_object_traverse},
1180
1181 /* as sequence */
1182 {Py_sq_length, mmap_length},
1183 {Py_sq_item, mmap_item},
1184 {Py_sq_ass_item, mmap_ass_item},
1185
1186 /* as mapping */
1187 {Py_mp_length, mmap_length},
1188 {Py_mp_subscript, mmap_subscript},
1189 {Py_mp_ass_subscript, mmap_ass_subscript},
1190
1191 /* as buffer */
1192 {Py_bf_getbuffer, mmap_buffer_getbuf},
1193 {Py_bf_releasebuffer, mmap_buffer_releasebuf},
1194 {0, NULL},
1195 };
1196
1197 static PyType_Spec mmap_object_spec = {
1198 .name = "mmap.mmap",
1199 .basicsize = sizeof(mmap_object),
1200 .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE |
1201 Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE),
1202 .slots = mmap_object_slots,
1203 };
1204
1205
1206 #ifdef UNIX
1207 #ifdef HAVE_LARGEFILE_SUPPORT
1208 #define _Py_PARSE_OFF_T "L"
1209 #else
1210 #define _Py_PARSE_OFF_T "l"
1211 #endif
1212
1213 static PyObject *
new_mmap_object(PyTypeObject * type,PyObject * args,PyObject * kwdict)1214 new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict)
1215 {
1216 struct _Py_stat_struct status;
1217 int fstat_result = -1;
1218 mmap_object *m_obj;
1219 Py_ssize_t map_size;
1220 off_t offset = 0;
1221 int fd, flags = MAP_SHARED, prot = PROT_WRITE | PROT_READ;
1222 int devzero = -1;
1223 int access = (int)ACCESS_DEFAULT;
1224 static char *keywords[] = {"fileno", "length",
1225 "flags", "prot",
1226 "access", "offset", NULL};
1227
1228 if (!PyArg_ParseTupleAndKeywords(args, kwdict, "in|iii" _Py_PARSE_OFF_T, keywords,
1229 &fd, &map_size, &flags, &prot,
1230 &access, &offset))
1231 return NULL;
1232 if (map_size < 0) {
1233 PyErr_SetString(PyExc_OverflowError,
1234 "memory mapped length must be positive");
1235 return NULL;
1236 }
1237 if (offset < 0) {
1238 PyErr_SetString(PyExc_OverflowError,
1239 "memory mapped offset must be positive");
1240 return NULL;
1241 }
1242
1243 if ((access != (int)ACCESS_DEFAULT) &&
1244 ((flags != MAP_SHARED) || (prot != (PROT_WRITE | PROT_READ))))
1245 return PyErr_Format(PyExc_ValueError,
1246 "mmap can't specify both access and flags, prot.");
1247 switch ((access_mode)access) {
1248 case ACCESS_READ:
1249 flags = MAP_SHARED;
1250 prot = PROT_READ;
1251 break;
1252 case ACCESS_WRITE:
1253 flags = MAP_SHARED;
1254 prot = PROT_READ | PROT_WRITE;
1255 break;
1256 case ACCESS_COPY:
1257 flags = MAP_PRIVATE;
1258 prot = PROT_READ | PROT_WRITE;
1259 break;
1260 case ACCESS_DEFAULT:
1261 /* map prot to access type */
1262 if ((prot & PROT_READ) && (prot & PROT_WRITE)) {
1263 /* ACCESS_DEFAULT */
1264 }
1265 else if (prot & PROT_WRITE) {
1266 access = ACCESS_WRITE;
1267 }
1268 else {
1269 access = ACCESS_READ;
1270 }
1271 break;
1272 default:
1273 return PyErr_Format(PyExc_ValueError,
1274 "mmap invalid access parameter.");
1275 }
1276
1277 if (PySys_Audit("mmap.__new__", "ini" _Py_PARSE_OFF_T,
1278 fd, map_size, access, offset) < 0) {
1279 return NULL;
1280 }
1281
1282 #ifdef __APPLE__
1283 /* Issue #11277: fsync(2) is not enough on OS X - a special, OS X specific
1284 fcntl(2) is necessary to force DISKSYNC and get around mmap(2) bug */
1285 if (fd != -1)
1286 (void)fcntl(fd, F_FULLFSYNC);
1287 #endif
1288
1289 if (fd != -1) {
1290 Py_BEGIN_ALLOW_THREADS
1291 fstat_result = _Py_fstat_noraise(fd, &status);
1292 Py_END_ALLOW_THREADS
1293 }
1294
1295 if (fd != -1 && fstat_result == 0 && S_ISREG(status.st_mode)) {
1296 if (map_size == 0) {
1297 if (status.st_size == 0) {
1298 PyErr_SetString(PyExc_ValueError,
1299 "cannot mmap an empty file");
1300 return NULL;
1301 }
1302 if (offset >= status.st_size) {
1303 PyErr_SetString(PyExc_ValueError,
1304 "mmap offset is greater than file size");
1305 return NULL;
1306 }
1307 if (status.st_size - offset > PY_SSIZE_T_MAX) {
1308 PyErr_SetString(PyExc_ValueError,
1309 "mmap length is too large");
1310 return NULL;
1311 }
1312 map_size = (Py_ssize_t) (status.st_size - offset);
1313 } else if (offset > status.st_size || status.st_size - offset < map_size) {
1314 PyErr_SetString(PyExc_ValueError,
1315 "mmap length is greater than file size");
1316 return NULL;
1317 }
1318 }
1319 m_obj = (mmap_object *)type->tp_alloc(type, 0);
1320 if (m_obj == NULL) {return NULL;}
1321 m_obj->data = NULL;
1322 m_obj->size = map_size;
1323 m_obj->pos = 0;
1324 m_obj->weakreflist = NULL;
1325 m_obj->exports = 0;
1326 m_obj->offset = offset;
1327 if (fd == -1) {
1328 m_obj->fd = -1;
1329 /* Assume the caller wants to map anonymous memory.
1330 This is the same behaviour as Windows. mmap.mmap(-1, size)
1331 on both Windows and Unix map anonymous memory.
1332 */
1333 #ifdef MAP_ANONYMOUS
1334 /* BSD way to map anonymous memory */
1335 flags |= MAP_ANONYMOUS;
1336
1337 /* VxWorks only supports MAP_ANONYMOUS with MAP_PRIVATE flag */
1338 #ifdef __VXWORKS__
1339 flags &= ~MAP_SHARED;
1340 flags |= MAP_PRIVATE;
1341 #endif
1342
1343 #else
1344 /* SVR4 method to map anonymous memory is to open /dev/zero */
1345 fd = devzero = _Py_open("/dev/zero", O_RDWR);
1346 if (devzero == -1) {
1347 Py_DECREF(m_obj);
1348 return NULL;
1349 }
1350 #endif
1351 }
1352 else {
1353 m_obj->fd = _Py_dup(fd);
1354 if (m_obj->fd == -1) {
1355 Py_DECREF(m_obj);
1356 return NULL;
1357 }
1358 }
1359
1360 m_obj->data = mmap(NULL, map_size,
1361 prot, flags,
1362 fd, offset);
1363
1364 if (devzero != -1) {
1365 close(devzero);
1366 }
1367
1368 if (m_obj->data == (char *)-1) {
1369 m_obj->data = NULL;
1370 Py_DECREF(m_obj);
1371 PyErr_SetFromErrno(PyExc_OSError);
1372 return NULL;
1373 }
1374 m_obj->access = (access_mode)access;
1375 return (PyObject *)m_obj;
1376 }
1377 #endif /* UNIX */
1378
1379 #ifdef MS_WINDOWS
1380
1381 /* A note on sizes and offsets: while the actual map size must hold in a
1382 Py_ssize_t, both the total file size and the start offset can be longer
1383 than a Py_ssize_t, so we use long long which is always 64-bit.
1384 */
1385
1386 static PyObject *
new_mmap_object(PyTypeObject * type,PyObject * args,PyObject * kwdict)1387 new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict)
1388 {
1389 mmap_object *m_obj;
1390 Py_ssize_t map_size;
1391 long long offset = 0, size;
1392 DWORD off_hi; /* upper 32 bits of offset */
1393 DWORD off_lo; /* lower 32 bits of offset */
1394 DWORD size_hi; /* upper 32 bits of size */
1395 DWORD size_lo; /* lower 32 bits of size */
1396 const char *tagname = "";
1397 DWORD dwErr = 0;
1398 int fileno;
1399 HANDLE fh = 0;
1400 int access = (access_mode)ACCESS_DEFAULT;
1401 DWORD flProtect, dwDesiredAccess;
1402 static char *keywords[] = { "fileno", "length",
1403 "tagname",
1404 "access", "offset", NULL };
1405
1406 if (!PyArg_ParseTupleAndKeywords(args, kwdict, "in|ziL", keywords,
1407 &fileno, &map_size,
1408 &tagname, &access, &offset)) {
1409 return NULL;
1410 }
1411
1412 if (PySys_Audit("mmap.__new__", "iniL",
1413 fileno, map_size, access, offset) < 0) {
1414 return NULL;
1415 }
1416
1417 switch((access_mode)access) {
1418 case ACCESS_READ:
1419 flProtect = PAGE_READONLY;
1420 dwDesiredAccess = FILE_MAP_READ;
1421 break;
1422 case ACCESS_DEFAULT: case ACCESS_WRITE:
1423 flProtect = PAGE_READWRITE;
1424 dwDesiredAccess = FILE_MAP_WRITE;
1425 break;
1426 case ACCESS_COPY:
1427 flProtect = PAGE_WRITECOPY;
1428 dwDesiredAccess = FILE_MAP_COPY;
1429 break;
1430 default:
1431 return PyErr_Format(PyExc_ValueError,
1432 "mmap invalid access parameter.");
1433 }
1434
1435 if (map_size < 0) {
1436 PyErr_SetString(PyExc_OverflowError,
1437 "memory mapped length must be positive");
1438 return NULL;
1439 }
1440 if (offset < 0) {
1441 PyErr_SetString(PyExc_OverflowError,
1442 "memory mapped offset must be positive");
1443 return NULL;
1444 }
1445
1446 /* assume -1 and 0 both mean invalid filedescriptor
1447 to 'anonymously' map memory.
1448 XXX: fileno == 0 is a valid fd, but was accepted prior to 2.5.
1449 XXX: Should this code be added?
1450 if (fileno == 0)
1451 PyErr_WarnEx(PyExc_DeprecationWarning,
1452 "don't use 0 for anonymous memory",
1453 1);
1454 */
1455 if (fileno != -1 && fileno != 0) {
1456 /* Ensure that fileno is within the CRT's valid range */
1457 fh = _Py_get_osfhandle(fileno);
1458 if (fh == INVALID_HANDLE_VALUE)
1459 return NULL;
1460
1461 /* Win9x appears to need us seeked to zero */
1462 lseek(fileno, 0, SEEK_SET);
1463 }
1464
1465 m_obj = (mmap_object *)type->tp_alloc(type, 0);
1466 if (m_obj == NULL)
1467 return NULL;
1468 /* Set every field to an invalid marker, so we can safely
1469 destruct the object in the face of failure */
1470 m_obj->data = NULL;
1471 m_obj->file_handle = INVALID_HANDLE_VALUE;
1472 m_obj->map_handle = NULL;
1473 m_obj->tagname = NULL;
1474 m_obj->offset = offset;
1475
1476 if (fh) {
1477 /* It is necessary to duplicate the handle, so the
1478 Python code can close it on us */
1479 if (!DuplicateHandle(
1480 GetCurrentProcess(), /* source process handle */
1481 fh, /* handle to be duplicated */
1482 GetCurrentProcess(), /* target proc handle */
1483 (LPHANDLE)&m_obj->file_handle, /* result */
1484 0, /* access - ignored due to options value */
1485 FALSE, /* inherited by child processes? */
1486 DUPLICATE_SAME_ACCESS)) { /* options */
1487 dwErr = GetLastError();
1488 Py_DECREF(m_obj);
1489 PyErr_SetFromWindowsErr(dwErr);
1490 return NULL;
1491 }
1492 if (!map_size) {
1493 DWORD low,high;
1494 low = GetFileSize(fh, &high);
1495 /* low might just happen to have the value INVALID_FILE_SIZE;
1496 so we need to check the last error also. */
1497 if (low == INVALID_FILE_SIZE &&
1498 (dwErr = GetLastError()) != NO_ERROR) {
1499 Py_DECREF(m_obj);
1500 return PyErr_SetFromWindowsErr(dwErr);
1501 }
1502
1503 size = (((long long) high) << 32) + low;
1504 if (size == 0) {
1505 PyErr_SetString(PyExc_ValueError,
1506 "cannot mmap an empty file");
1507 Py_DECREF(m_obj);
1508 return NULL;
1509 }
1510 if (offset >= size) {
1511 PyErr_SetString(PyExc_ValueError,
1512 "mmap offset is greater than file size");
1513 Py_DECREF(m_obj);
1514 return NULL;
1515 }
1516 if (size - offset > PY_SSIZE_T_MAX) {
1517 PyErr_SetString(PyExc_ValueError,
1518 "mmap length is too large");
1519 Py_DECREF(m_obj);
1520 return NULL;
1521 }
1522 m_obj->size = (Py_ssize_t) (size - offset);
1523 } else {
1524 m_obj->size = map_size;
1525 size = offset + map_size;
1526 }
1527 }
1528 else {
1529 m_obj->size = map_size;
1530 size = offset + map_size;
1531 }
1532
1533 /* set the initial position */
1534 m_obj->pos = (size_t) 0;
1535
1536 m_obj->weakreflist = NULL;
1537 m_obj->exports = 0;
1538 /* set the tag name */
1539 if (tagname != NULL && *tagname != '\0') {
1540 m_obj->tagname = PyMem_Malloc(strlen(tagname)+1);
1541 if (m_obj->tagname == NULL) {
1542 PyErr_NoMemory();
1543 Py_DECREF(m_obj);
1544 return NULL;
1545 }
1546 strcpy(m_obj->tagname, tagname);
1547 }
1548 else
1549 m_obj->tagname = NULL;
1550
1551 m_obj->access = (access_mode)access;
1552 size_hi = (DWORD)(size >> 32);
1553 size_lo = (DWORD)(size & 0xFFFFFFFF);
1554 off_hi = (DWORD)(offset >> 32);
1555 off_lo = (DWORD)(offset & 0xFFFFFFFF);
1556 /* For files, it would be sufficient to pass 0 as size.
1557 For anonymous maps, we have to pass the size explicitly. */
1558 m_obj->map_handle = CreateFileMapping(m_obj->file_handle,
1559 NULL,
1560 flProtect,
1561 size_hi,
1562 size_lo,
1563 m_obj->tagname);
1564 if (m_obj->map_handle != NULL) {
1565 m_obj->data = (char *) MapViewOfFile(m_obj->map_handle,
1566 dwDesiredAccess,
1567 off_hi,
1568 off_lo,
1569 m_obj->size);
1570 if (m_obj->data != NULL)
1571 return (PyObject *)m_obj;
1572 else {
1573 dwErr = GetLastError();
1574 CloseHandle(m_obj->map_handle);
1575 m_obj->map_handle = NULL;
1576 }
1577 } else
1578 dwErr = GetLastError();
1579 Py_DECREF(m_obj);
1580 PyErr_SetFromWindowsErr(dwErr);
1581 return NULL;
1582 }
1583 #endif /* MS_WINDOWS */
1584
1585 static int
mmap_traverse(PyObject * module,visitproc visit,void * arg)1586 mmap_traverse(PyObject *module, visitproc visit, void *arg)
1587 {
1588 mmap_state *state = get_mmap_state(module);
1589 Py_VISIT(state->mmap_object_type);
1590 return 0;
1591 }
1592
1593 static int
mmap_clear(PyObject * module)1594 mmap_clear(PyObject *module)
1595 {
1596 mmap_state *state = get_mmap_state(module);
1597 Py_CLEAR(state->mmap_object_type);
1598 return 0;
1599 }
1600
1601 static void
mmap_free(void * module)1602 mmap_free(void *module)
1603 {
1604 mmap_clear((PyObject *)module);
1605 }
1606
1607 static int
mmap_exec(PyObject * module)1608 mmap_exec(PyObject *module)
1609 {
1610 mmap_state *state = get_mmap_state(module);
1611
1612 Py_INCREF(PyExc_OSError);
1613 if (PyModule_AddObject(module, "error", PyExc_OSError) < 0) {
1614 Py_DECREF(PyExc_OSError);
1615 return -1;
1616 }
1617
1618 state->mmap_object_type = (PyTypeObject *)PyType_FromModuleAndSpec(module,
1619 &mmap_object_spec,
1620 NULL);
1621 if (state->mmap_object_type == NULL) {
1622 return -1;
1623 }
1624 if (PyModule_AddType(module, state->mmap_object_type) < 0) {
1625 return -1;
1626 }
1627
1628 #define ADD_INT_MACRO(module, constant) \
1629 do { \
1630 if (PyModule_AddIntConstant(module, #constant, constant) < 0) { \
1631 return -1; \
1632 } \
1633 } while (0)
1634
1635 #ifdef PROT_EXEC
1636 ADD_INT_MACRO(module, PROT_EXEC);
1637 #endif
1638 #ifdef PROT_READ
1639 ADD_INT_MACRO(module, PROT_READ);
1640 #endif
1641 #ifdef PROT_WRITE
1642 ADD_INT_MACRO(module, PROT_WRITE);
1643 #endif
1644
1645 #ifdef MAP_SHARED
1646 ADD_INT_MACRO(module, MAP_SHARED);
1647 #endif
1648 #ifdef MAP_PRIVATE
1649 ADD_INT_MACRO(module, MAP_PRIVATE);
1650 #endif
1651 #ifdef MAP_DENYWRITE
1652 ADD_INT_MACRO(module, MAP_DENYWRITE);
1653 #endif
1654 #ifdef MAP_EXECUTABLE
1655 ADD_INT_MACRO(module, MAP_EXECUTABLE);
1656 #endif
1657 #ifdef MAP_ANONYMOUS
1658 if (PyModule_AddIntConstant(module, "MAP_ANON", MAP_ANONYMOUS) < 0 ) {
1659 return -1;
1660 }
1661 ADD_INT_MACRO(module, MAP_ANONYMOUS);
1662 #endif
1663 #ifdef MAP_POPULATE
1664 ADD_INT_MACRO(module, MAP_POPULATE);
1665 #endif
1666 #ifdef MAP_STACK
1667 // Mostly a no-op on Linux and NetBSD, but useful on OpenBSD
1668 // for stack usage (even on x86 arch)
1669 ADD_INT_MACRO(module, MAP_STACK);
1670 #endif
1671 if (PyModule_AddIntConstant(module, "PAGESIZE", (long)my_getpagesize()) < 0 ) {
1672 return -1;
1673 }
1674
1675 if (PyModule_AddIntConstant(module, "ALLOCATIONGRANULARITY", (long)my_getallocationgranularity()) < 0 ) {
1676 return -1;
1677 }
1678
1679 ADD_INT_MACRO(module, ACCESS_DEFAULT);
1680 ADD_INT_MACRO(module, ACCESS_READ);
1681 ADD_INT_MACRO(module, ACCESS_WRITE);
1682 ADD_INT_MACRO(module, ACCESS_COPY);
1683
1684 #ifdef HAVE_MADVISE
1685 // Conventional advice values
1686 #ifdef MADV_NORMAL
1687 ADD_INT_MACRO(module, MADV_NORMAL);
1688 #endif
1689 #ifdef MADV_RANDOM
1690 ADD_INT_MACRO(module, MADV_RANDOM);
1691 #endif
1692 #ifdef MADV_SEQUENTIAL
1693 ADD_INT_MACRO(module, MADV_SEQUENTIAL);
1694 #endif
1695 #ifdef MADV_WILLNEED
1696 ADD_INT_MACRO(module, MADV_WILLNEED);
1697 #endif
1698 #ifdef MADV_DONTNEED
1699 ADD_INT_MACRO(module, MADV_DONTNEED);
1700 #endif
1701
1702 // Linux-specific advice values
1703 #ifdef MADV_REMOVE
1704 ADD_INT_MACRO(module, MADV_REMOVE);
1705 #endif
1706 #ifdef MADV_DONTFORK
1707 ADD_INT_MACRO(module, MADV_DONTFORK);
1708 #endif
1709 #ifdef MADV_DOFORK
1710 ADD_INT_MACRO(module, MADV_DOFORK);
1711 #endif
1712 #ifdef MADV_HWPOISON
1713 ADD_INT_MACRO(module, MADV_HWPOISON);
1714 #endif
1715 #ifdef MADV_MERGEABLE
1716 ADD_INT_MACRO(module, MADV_MERGEABLE);
1717 #endif
1718 #ifdef MADV_UNMERGEABLE
1719 ADD_INT_MACRO(module, MADV_UNMERGEABLE);
1720 #endif
1721 #ifdef MADV_SOFT_OFFLINE
1722 ADD_INT_MACRO(module, MADV_SOFT_OFFLINE);
1723 #endif
1724 #ifdef MADV_HUGEPAGE
1725 ADD_INT_MACRO(module, MADV_HUGEPAGE);
1726 #endif
1727 #ifdef MADV_NOHUGEPAGE
1728 ADD_INT_MACRO(module, MADV_NOHUGEPAGE);
1729 #endif
1730 #ifdef MADV_DONTDUMP
1731 ADD_INT_MACRO(module, MADV_DONTDUMP);
1732 #endif
1733 #ifdef MADV_DODUMP
1734 ADD_INT_MACRO(module, MADV_DODUMP);
1735 #endif
1736 #ifdef MADV_FREE // (Also present on FreeBSD and macOS.)
1737 ADD_INT_MACRO(module, MADV_FREE);
1738 #endif
1739
1740 // FreeBSD-specific
1741 #ifdef MADV_NOSYNC
1742 ADD_INT_MACRO(module, MADV_NOSYNC);
1743 #endif
1744 #ifdef MADV_AUTOSYNC
1745 ADD_INT_MACRO(module, MADV_AUTOSYNC);
1746 #endif
1747 #ifdef MADV_NOCORE
1748 ADD_INT_MACRO(module, MADV_NOCORE);
1749 #endif
1750 #ifdef MADV_CORE
1751 ADD_INT_MACRO(module, MADV_CORE);
1752 #endif
1753 #ifdef MADV_PROTECT
1754 ADD_INT_MACRO(module, MADV_PROTECT);
1755 #endif
1756
1757 // Darwin-specific
1758 #ifdef MADV_FREE_REUSABLE // (As MADV_FREE but reclaims more faithful for task_info/Activity Monitor...)
1759 ADD_INT_MACRO(module, MADV_FREE_REUSABLE);
1760 #endif
1761 #ifdef MADV_FREE_REUSE // (Reuse pages previously tagged as reusable)
1762 ADD_INT_MACRO(module, MADV_FREE_REUSE);
1763 #endif
1764 #endif // HAVE_MADVISE
1765 return 0;
1766 }
1767
1768 static PyModuleDef_Slot mmap_slots[] = {
1769 {Py_mod_exec, mmap_exec},
1770 {0, NULL}
1771 };
1772
1773 static struct PyModuleDef mmapmodule = {
1774 PyModuleDef_HEAD_INIT,
1775 .m_name = "mmap",
1776 .m_size = sizeof(mmap_state),
1777 .m_slots = mmap_slots,
1778 .m_traverse = mmap_traverse,
1779 .m_clear = mmap_clear,
1780 .m_free = mmap_free,
1781 };
1782
1783 PyMODINIT_FUNC
PyInit_mmap(void)1784 PyInit_mmap(void)
1785 {
1786 return PyModuleDef_Init(&mmapmodule);
1787 }
1788