[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[Emacs-diffs] trunk r114578: Do not allocate huge temporary memory areas
From: |
Dmitry Antipov |
Subject: |
[Emacs-diffs] trunk r114578: Do not allocate huge temporary memory areas and objects while encoding |
Date: |
Tue, 08 Oct 2013 06:40:52 +0000 |
User-agent: |
Bazaar (2.6b2) |
------------------------------------------------------------
revno: 114578
revision-id: address@hidden
parent: address@hidden
committer: Dmitry Antipov <address@hidden>
branch nick: trunk
timestamp: Tue 2013-10-08 10:40:09 +0400
message:
Do not allocate huge temporary memory areas and objects while encoding
for file I/O, thus reducing an enormous memory usage for large buffers.
See http://lists.gnu.org/archive/html/emacs-devel/2013-10/msg00180.html.
* coding.h (struct coding_system): New member raw_destination.
* coding.c (setup_coding_system): Initialize it to zero.
(encode_coding_object): If raw_destination is set, do not create
dst_object. Add comment.
* fileio.c (toplevel): New constant E_WRITE_MAX.
(e_write): Do not encode more than E_WRITE_MAX characters per one loop
iteration. Use raw_destination if E_WRITE_MAX characters is encoded.
modified:
src/ChangeLog changelog-20091113204419-o5vbwnq5f7feedwu-1438
src/coding.c coding.c-20091113204419-o5vbwnq5f7feedwu-1077
src/coding.h coding.h-20091113204419-o5vbwnq5f7feedwu-1078
src/fileio.c fileio.c-20091113204419-o5vbwnq5f7feedwu-210
=== modified file 'src/ChangeLog'
--- a/src/ChangeLog 2013-10-08 06:12:40 +0000
+++ b/src/ChangeLog 2013-10-08 06:40:09 +0000
@@ -1,3 +1,16 @@
+2013-10-08 Dmitry Antipov <address@hidden>
+
+ Do not allocate huge temporary memory areas and objects while encoding
+ for file I/O, thus reducing an enormous memory usage for large buffers.
+ See http://lists.gnu.org/archive/html/emacs-devel/2013-10/msg00180.html.
+ * coding.h (struct coding_system): New member raw_destination.
+ * coding.c (setup_coding_system): Initialize it to zero.
+ (encode_coding_object): If raw_destination is set, do not create
+ dst_object. Add comment.
+ * fileio.c (toplevel): New constant E_WRITE_MAX.
+ (e_write): Do not encode more than E_WRITE_MAX characters per one loop
+ iteration. Use raw_destination if E_WRITE_MAX characters is encoded.
+
2013-10-08 Jan Djärv <address@hidden>
* nsterm.m (windowDidExitFullScreen:):
=== modified file 'src/coding.c'
--- a/src/coding.c 2013-08-26 05:20:59 +0000
+++ b/src/coding.c 2013-10-08 06:40:09 +0000
@@ -5761,6 +5761,7 @@
coding->safe_charsets = SDATA (val);
coding->default_char = XINT (CODING_ATTR_DEFAULT_CHAR (attrs));
coding->carryover_bytes = 0;
+ coding->raw_destination = 0;
coding_type = CODING_ATTR_TYPE (attrs);
if (EQ (coding_type, Qundecided))
@@ -8352,6 +8353,11 @@
{
if (BUFFERP (coding->dst_object))
coding->dst_object = Fbuffer_string ();
+ else if (coding->raw_destination)
+ /* This is used to avoid creating huge Lisp string.
+ NOTE: caller who sets `raw_destination' is also
+ responsible for freeing `destination' buffer. */
+ coding->dst_object = Qnil;
else
{
coding->dst_object
=== modified file 'src/coding.h'
--- a/src/coding.h 2013-08-30 12:17:44 +0000
+++ b/src/coding.h 2013-10-08 06:40:09 +0000
@@ -512,6 +512,10 @@
`charbuf', but at `src_object'. */
unsigned chars_at_source : 1;
+ /* Nonzero if the result of conversion is in `destination'
+ buffer rather than in `dst_object'. */
+ unsigned raw_destination : 1;
+
/* Set to 1 if charbuf contains an annotation. */
unsigned annotated : 1;
=== modified file 'src/fileio.c'
--- a/src/fileio.c 2013-09-11 05:03:23 +0000
+++ b/src/fileio.c 2013-10-08 06:40:09 +0000
@@ -5263,6 +5263,10 @@
return 1;
}
+/* Maximum number of characters that the next
+ function encodes per one loop iteration. */
+
+enum { E_WRITE_MAX = 8 * 1024 * 1024 };
/* Write text in the range START and END into descriptor DESC,
encoding them with coding system CODING. If STRING is nil, START
@@ -5289,9 +5293,16 @@
coding->src_multibyte = SCHARS (string) < SBYTES (string);
if (CODING_REQUIRE_ENCODING (coding))
{
- encode_coding_object (coding, string,
- start, string_char_to_byte (string, start),
- end, string_char_to_byte (string, end), Qt);
+ ptrdiff_t nchars = min (end - start, E_WRITE_MAX);
+
+ /* Avoid creating huge Lisp string in encode_coding_object. */
+ if (nchars == E_WRITE_MAX)
+ coding->raw_destination = 1;
+
+ encode_coding_object
+ (coding, string, start, string_char_to_byte (string, start),
+ start + nchars, string_char_to_byte (string, start + nchars),
+ Qt);
}
else
{
@@ -5308,8 +5319,15 @@
coding->src_multibyte = (end - start) < (end_byte - start_byte);
if (CODING_REQUIRE_ENCODING (coding))
{
- encode_coding_object (coding, Fcurrent_buffer (),
- start, start_byte, end, end_byte, Qt);
+ ptrdiff_t nchars = min (end - start, E_WRITE_MAX);
+
+ /* Likewise. */
+ if (nchars == E_WRITE_MAX)
+ coding->raw_destination = 1;
+
+ encode_coding_object
+ (coding, Fcurrent_buffer (), start, start_byte,
+ start + nchars, CHAR_TO_BYTE (start + nchars), Qt);
}
else
{
@@ -5330,11 +5348,19 @@
if (coding->produced > 0)
{
- char *buf = (STRINGP (coding->dst_object)
- ? SSDATA (coding->dst_object)
- : (char *) BYTE_POS_ADDR (coding->dst_pos_byte));
+ char *buf = (coding->raw_destination ? (char *) coding->destination
+ : (STRINGP (coding->dst_object)
+ ? SSDATA (coding->dst_object)
+ : (char *) BYTE_POS_ADDR (coding->dst_pos_byte)));
coding->produced -= emacs_write_sig (desc, buf, coding->produced);
+ if (coding->raw_destination)
+ {
+ /* We're responsible for freeing this, see
+ encode_coding_object to check why. */
+ xfree (coding->destination);
+ coding->raw_destination = 0;
+ }
if (coding->produced)
return 0;
}
[Prev in Thread] |
Current Thread |
[Next in Thread] |
- [Emacs-diffs] trunk r114578: Do not allocate huge temporary memory areas and objects while encoding,
Dmitry Antipov <=