[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
Re: [bug-gettext] [PATCH] its: Add new preserveSpaceRule "paragraph"
From: |
Daiki Ueno |
Subject: |
Re: [bug-gettext] [PATCH] its: Add new preserveSpaceRule "paragraph" |
Date: |
Thu, 14 Feb 2019 12:39:52 +0100 |
User-agent: |
Gnus/5.13 (Gnus v5.13) Emacs/27.0.50 (gnu/linux) |
Hello Bruno,
Bruno Haible <address@hidden> writes:
>> + /* Normalize whitespaces in the paragraph. */
>> + assert (pend != NULL);
>> + c = *pend;
>> + *pend = '\0';
>> + for (pp = p; *pp != '\0';)
>> + {
>> + size_t len = strspn (pp, " \t\n");
>> + if (len > 0)
>> + {
>> + *pp = ' ';
>> + memmove (pp + 1, pp + len, end - (pp + len));
>> + end -= len - 1;
>> + *end = '\0';
>> + pend -= len - 1;
>> + *pend = '\0';
>> + pp++;
>> + }
>> + pp += strcspn (pp, " \t\n");
>> + }
>> + *pend = c;
>> + p = pp + strspn (pend, "\n");
>> + }
>> + return result;
>
> This loop is O(N*M), where N is the number of whitespace blocks in the
> current paragraph and M is the number of characters starting at 'p'
> (because the memmove extends from the current paragraph to the end of
> the string, and there are as many memmove invocations as there are
> whitespace blocks.
>
> Could it be rewritten as an O(N+M) loop, by using two pointers ip
> (input pointer) and op (output pointer), instead of pp,
> that both run starting at p and satisfy the inequalities
> p <= op <= ip <= pend
> ?
Thank you for the review, that's a good point. I have rewritten the
loop with the parallel pointers instead of memmove as attached. As a
bonus, the behavior becomes closer to intltool: now it can normalize
paragraph boundaries with "\n\n".
Regards,
--
Daiki Ueno
>From f25f516d3d7814dd3de86c281fe30f15eac832d7 Mon Sep 17 00:00:00 2001
From: Daiki Ueno <address@hidden>
Date: Mon, 11 Feb 2019 11:26:53 +0100
Subject: [PATCH] its: Add new preserveSpaceRule "paragraph"
This implements a new text extraction rule preserving paragraph
boundaries, as mentioned in:
https://gitlab.gnome.org/GNOME/glib/issues/1350
* gettext-tools/doc/gettext.texi (Preparing ITS Rules): Mention "paragraph".
* gettext-tools/src/its.c (its_rule_list_extract_text): Accept "paragraph".
(its_merge_context_merge_node): Likewise.
(normalize_whitespace): Handle "paragraph" rule.
* gettext-tools/src/its.h (ITS_WHITESPACE_NORMALIZE_PARAGRAPH): New enum value.
* gettext-tools/tests/xgettext-its-1: Add test for "paragraph" rule.
---
gettext-tools/doc/gettext.texi | 12 +++--
gettext-tools/src/its.c | 73 +++++++++++++++++++++++++++++-
gettext-tools/src/its.h | 1 +
gettext-tools/tests/xgettext-its-1 | 44 ++++++++++++++++++
4 files changed, 124 insertions(+), 6 deletions(-)
diff --git a/gettext-tools/doc/gettext.texi b/gettext-tools/doc/gettext.texi
index 892854c75..306605842 100644
--- a/gettext-tools/doc/gettext.texi
+++ b/gettext-tools/doc/gettext.texi
@@ -12354,10 +12354,12 @@ A required @code{escape} attribute with the value
@code{yes} or @code{no}.
@item Extended Preserve Space
This data category extends the standard @samp{Preserve Space} data
-category with the additional value @samp{trim}. The value means to
-remove the leading and trailing whitespaces of the content, but not to
-normalize whitespaces in the middle. In the global rule, the
address@hidden element contains the following:
+category with the additional values @samp{trim} and @samp{paragraph}.
address@hidden means to remove the leading and trailing whitespaces of the
+content, but not to normalize whitespaces in the middle.
address@hidden means to normalize the content but keep the paragraph
+boundaries. In the global
+rule, the @code{preserveSpaceRule} element contains the following:
@itemize
@item
@@ -12366,7 +12368,7 @@ that selects the nodes to which this rule applies.
@item
A required @code{space} attribute with the value @code{default},
address@hidden, or @code{trim}.
address@hidden, @code{trim}, or @code{paragraph}.
@end itemize
@end table
diff --git a/gettext-tools/src/its.c b/gettext-tools/src/its.c
index 9b4b397e5..cdcb2efb7 100644
--- a/gettext-tools/src/its.c
+++ b/gettext-tools/src/its.c
@@ -399,6 +399,69 @@ normalize_whitespace (const char *text, enum
its_whitespace_type_ty whitespace)
case ITS_WHITESPACE_TRIM:
return trim (text);
+ case ITS_WHITESPACE_NORMALIZE_PARAGRAPH:
+ /* Normalize whitespaces within the text, keeping paragraph
+ boundaries. */
+ {
+ char *result, *p, *out;
+
+ result = trim (text);
+ for (p = out = result; *p != '\0';)
+ {
+ char *pp, *pend = NULL, *next = NULL;
+ bool last_ws = false;
+
+ /* Find a paragraph boundary. */
+ for (pp = p; *pp != '\0';)
+ {
+ char *nl = strchrnul (pp, '\n');
+ if (*nl == '\0')
+ {
+ pend = nl;
+ next = pend;
+ break;
+ }
+ pp = nl + 1;
+ pp += strspn (pp, " \t\n");
+ if (*pp == '\n')
+ {
+ pend = nl;
+ next = pp + 1;
+ break;
+ }
+ }
+
+ /* Normalize whitespaces in the paragraph. */
+ assert (pend != NULL);
+ for (pp = p; pp < pend; pp++)
+ if (!(*pp == ' ' || *pp == '\t' || *pp == '\n'))
+ break;
+ for (; pp < pend; pp++)
+ {
+ if (*pp == ' ' || *pp == '\t' || *pp == '\n')
+ {
+ if (!last_ws)
+ {
+ *out++ = ' ';
+ last_ws = true;
+ }
+ }
+ else
+ {
+ *out++ = *pp;
+ last_ws = false;
+ }
+ }
+ if (*pend != '\0')
+ {
+ memcpy (out, "\n\n", 2);
+ out += 2;
+ }
+ p = next;
+ }
+ *out = '\0';
+ return result;
+ }
default:
/* Normalize whitespaces within the text, but not at the beginning
nor the end of the text. */
@@ -996,7 +1059,11 @@ its_preserve_space_rule_constructor (struct its_rule_ty
*pop,
|| strcmp (prop, "default") == 0
/* gettext extension: remove leading/trailing whitespaces only. */
|| (node->ns && xmlStrEqual (node->ns->href, BAD_CAST GT_NS)
- && strcmp (prop, "trim") == 0)))
+ && strcmp (prop, "trim") == 0)
+ /* gettext extension: same as default except keeping
+ paragraph boundaries. */
+ || (node->ns && xmlStrEqual (node->ns->href, BAD_CAST GT_NS)
+ && strcmp (prop, "paragraph") == 0)))
{
error (0, 0, _("invalid attribute value \"%s\" for \"%s\""),
prop, "space");
@@ -1715,6 +1782,8 @@ its_rule_list_extract_text (its_rule_list_ty *rules,
whitespace = ITS_WHITESPACE_PRESERVE;
else if (value && strcmp (value, "trim") == 0)
whitespace = ITS_WHITESPACE_TRIM;
+ else if (value && strcmp (value, "paragraph") == 0)
+ whitespace = ITS_WHITESPACE_NORMALIZE_PARAGRAPH;
else
whitespace = ITS_WHITESPACE_NORMALIZE;
@@ -1842,6 +1911,8 @@ its_merge_context_merge_node (struct its_merge_context_ty
*context,
whitespace = ITS_WHITESPACE_PRESERVE;
else if (value && strcmp (value, "trim") == 0)
whitespace = ITS_WHITESPACE_TRIM;
+ else if (value && strcmp (value, "paragraph") == 0)
+ whitespace = ITS_WHITESPACE_NORMALIZE_PARAGRAPH;
else
whitespace = ITS_WHITESPACE_NORMALIZE;
diff --git a/gettext-tools/src/its.h b/gettext-tools/src/its.h
index 72c30c992..49af5cec5 100644
--- a/gettext-tools/src/its.h
+++ b/gettext-tools/src/its.h
@@ -33,6 +33,7 @@ enum its_whitespace_type_ty
{
ITS_WHITESPACE_PRESERVE,
ITS_WHITESPACE_NORMALIZE,
+ ITS_WHITESPACE_NORMALIZE_PARAGRAPH,
ITS_WHITESPACE_TRIM
};
diff --git a/gettext-tools/tests/xgettext-its-1
b/gettext-tools/tests/xgettext-its-1
index 125f3e682..975a547cd 100755
--- a/gettext-tools/tests/xgettext-its-1
+++ b/gettext-tools/tests/xgettext-its-1
@@ -171,6 +171,30 @@ cat <<\EOF >messages.xml
<message unescaped="This is an unescaped attribute <>&"">
<p></p>
</message>
+ <message>
+ <p xml:space="paragraph">
+ This is the first paragraph with
+a newline.
+
+ This is the second paragprah with spaces.
+
+
+ This is the last paragraph.</p>
+ </message>
+ <message>
+ <p xml:space="paragraph">This is the only one paragraph</p>
+ </message>
+ <message>
+ <p xml:space="paragraph">This is the only one paragraph with a boundary
+
+</p>
+ </message>
+ <message>
+ <p xml:space="paragraph"></p>
+ </message>
+ <message>
+ <p xml:space="paragraph"> </p>
+ </message>
</messages>
EOF
@@ -247,6 +271,26 @@ msgstr ""
#: messages.xml:61
msgid "This is an unescaped attribute <>&\""
msgstr ""
+
+#. (itstool) path: message/p
+#: messages.xml:65
+msgid ""
+"This is the first paragraph with a newline.\n"
+"\n"
+"This is the second paragprah with spaces.\n"
+"\n"
+"This is the last paragraph."
+msgstr ""
+
+#. (itstool) path: message/p
+#: messages.xml:75
+msgid "This is the only one paragraph"
+msgstr ""
+
+#. (itstool) path: message/p
+#: messages.xml:78
+msgid "This is the only one paragraph with a boundary"
+msgstr ""
EOF
: ${DIFF=diff}
--
2.20.1