branch master updated: use parse_texi_line() more in tests when relevant

[Patrice Dumas]

This is an automated email from the git hooks/post-receive script.

pertusus pushed a commit to branch master
in repository texinfo.

The following commit(s) were added to refs/heads/master by this push:
     new 9d6fef944d use parse_texi_line() more in tests when relevant
9d6fef944d is described below

commit 9d6fef944d117356fce39d691a7b6cc1d71b2b7f
Author: Patrice Dumas <pertusus@free.fr>
AuthorDate: Sun Jan 9 17:27:30 2022 +0100

    use parse_texi_line() more in tests when relevant
---
 tp/t/accents.t                      |  8 +++---
 tp/t/nodenormalization.t            | 53 +++++++++++++++++++++++++-----------
 tp/t/protect_character_in_texinfo.t | 54 ++++++++++++++++++++++++-------------
 tp/t/test_brace_count.t             | 44 +++++++++++++++++++++++-------
 tp/t/test_is_content_empty.t        | 17 +++++++-----
 tp/t/test_utils.pl                  |  2 +-
 6 files changed, 125 insertions(+), 53 deletions(-)

diff --git a/tp/t/accents.t b/tp/t/accents.t
index d9fdb71c17..1a707ebbb0 100644
--- a/tp/t/accents.t
+++ b/tp/t/accents.t
@@ -22,7 +22,9 @@ sub _find_accent($)
 {
   my $root = shift;
   my $current = $root;
-  while ($current->{'type'} and ($current->{'type'} eq 'before_node_section'
+  while ($current->{'type'} and ($current->{'type'} eq 'root_line'
+                           # following could be useful if parse_texi_text is 
used
+                                 or $current->{'type'} eq 'before_node_section'
                                  or $current->{'type'} eq 'document_root'
                                  or $current->{'type'} eq 'paragraph')) {
     $current = $current->{'contents'}->[0];
@@ -37,7 +39,7 @@ sub test_accent_stack ($)
   my $name = $test->[1];
   my $reference = $test->[2];
   my $parser = Texinfo::Parser::parser();
-  my $root = $parser->parse_texi_text($texi);
+  my $root = $parser->parse_texi_line($texi);
   my $accent_tree = _find_accent($root);
   my ($contents, $commands_stack) = 
     Texinfo::Convert::Utils::find_innermost_accent_contents($accent_tree);
@@ -86,7 +88,7 @@ sub test_enable_encoding ($)
   my $reference_xml_numeric_entity = $test->[4];
   my $reference_unicode = $test->[5];
   my $parser = Texinfo::Parser::parser();
-  my $root = $parser->parse_texi_text($texi);
+  my $root = $parser->parse_texi_line($texi);
   my $accent_tree = _find_accent($root);
 
   my ($contents, $commands_stack) = 
diff --git a/tp/t/nodenormalization.t b/tp/t/nodenormalization.t
index 8f34f45d0d..d52f867d40 100644
--- a/tp/t/nodenormalization.t
+++ b/tp/t/nodenormalization.t
@@ -5,7 +5,7 @@ use Texinfo::ModulePath (undef, undef, 'updirs' => 2);
 
 use Test::More;
 
-BEGIN { plan tests => 8; }
+BEGIN { plan tests => 10; }
 
 use Data::Dumper;
 use Locale::Messages;
@@ -14,9 +14,6 @@ use Texinfo::Convert::Texinfo;
 use Texinfo::Convert::NodeNameNormalization qw(normalize_node 
transliterate_texinfo);
 use Texinfo::Parser;
 
-# Currently, tests check that NodeNameNormalization do not break with complete 
-# Texinfo trees, not that the output is correct.
-
 my $srcdir = $ENV{'srcdir'};
 if (defined($srcdir)) {
   $srcdir =~ s/\/*$/\//;
@@ -32,9 +29,12 @@ my $parser = Texinfo::Parser::parser({'INCLUDE_DIRECTORIES' 
=> [
                                           $srcdir.'t/include/'],
                                       'EXPANDED_FORMATS' => ['html', 'tex']});
 
+# Check that NodeNameNormalization do not break with complete
+# Texinfo trees.  Not much check of the output for these tests.
+
 my $invalid_line = '@noindent Text @titlefont{in titlefont} @anchor{in 
anchor}@footnote{footnote} @exdent exdent';
 
-my $tree = $parser->parse_texi_text($invalid_line);
+my $tree = $parser->parse_texi_line($invalid_line);
 #print STDERR Data::Dumper->Dump([$tree]);
 my $normalized_invalid = normalize_node($tree);
 #print STDERR "Invalid: $normalized_invalid\n";
@@ -137,12 +137,12 @@ in float
 @bye
 ';
 
-my $manual_tree = $parser->parse_texi_text($texinfo_manual);
-my $check_texinfo = 
Texinfo::Convert::Texinfo::convert_to_texinfo($manual_tree);
-is ($texinfo_manual, $check_texinfo, 'check manual parsing');
+my $texinfo_manual_tree = $parser->parse_texi_text($texinfo_manual);
+my $check_texinfo = 
Texinfo::Convert::Texinfo::convert_to_texinfo($texinfo_manual_tree);
+is ($texinfo_manual, $check_texinfo, 'check parsing of a manual');
 
-#print STDERR Data::Dumper->Dump([$manual_tree]);
-my $normalized_manual = normalize_node($manual_tree);
+#print STDERR Data::Dumper->Dump([$texinfo_manual_tree]);
+my $normalized_manual = normalize_node($texinfo_manual_tree);
 #print STDERR "Manual: $normalized_manual\n";
 
 ok($normalized_manual =~ /^[\w\-]+$/, 'normalized tree is a valid id');
@@ -151,13 +151,13 @@ ok($normalized_manual =~ /^[\w\-]+$/, 'normalized tree is 
a valid id');
 
 my $texi_line = 'A @sc{sc} accents @"i @"{@dotless{i}} @`{@=E} @l{} @,{@\'C} 
@={@,{@~{n}}} @v{@\'{r}} @={@~{@dotless{i}}} @"y @dotless{i} @dotless{j} @,{C} 
@ogonek{E} @udotaccent{a} @tieaccent{a} @dotaccent{a} characters @l{} 
@exclamdown{} @aa{} @oe{} @comma{} @error{} @today{} @dots{} @enddots{} no 
brace commands @@ @: @. @  @* @} signs  -- --- `` \'\' !_"#$%&\'()*+-. 
/;<=>?[\\]^_`|~';
 
-my $line_tree = $parser->parse_texi_text($texi_line);
+my $line_tree = $parser->parse_texi_line($texi_line);
 my $normalized_line = normalize_node($line_tree);
-is ($normalized_line, 
+is ($normalized_line,
 
'A-SC-accents-_00ef-_00ef-_1e14-_0142-_1e08-_0146_0303_0304-_0155_030c-_0129_0304-_00ff-_0131-j-_00c7-_0118-_1ea1-a_0361-_0227-characters-_0142-_00a1-_00e5-_0153-_002c-error_002d_002d_003e--_2026-_002e_002e_002e-no-brace-commands-_0040--_002e-----_007d-signs-_002d_002d-_002d_002d_002d-_0060_0060-_0027_0027-_0021_005f_0022_0023_0024_0025_0026_0027_0028_0029_002a_002b_002d_002e-_002f_003b_003c_003d_003e_003f_005b_005c_005d_005e_005f_0060_007c_007e',
   'normalized complex line');
 my $transliterated_line = transliterate_texinfo($line_tree);
-is ($transliterated_line, 
+is ($transliterated_line,
 
'A-SC-accents-i-i-E-l-C-n-r-i-y-i-j-C-E-a-a-a-characters-l-_00a1-aa-oe-_002c-error_002d_002d_003e--_2026-_002e_002e_002e-no-brace-commands-_0040--_002e-----_007d-signs-_002d_002d-_002d_002d_002d-_0060_0060-_0027_0027-_0021_005f_0022_0023_0024_0025_0026_0027_0028_0029_002a_002b_002d_002e-_002f_003b_003c_003d_003e_003f_005b_005c_005d_005e_005f_0060_007c_007e',
  'transliterated complex line');
 my $transliterated_line_no_unidecode = transliterate_texinfo($line_tree, 1);
@@ -165,12 +165,33 @@ is ($transliterated_line_no_unidecode,
 
'A-SC-accents-i-i-_1e14-l-_1e08-n-r-i-y-_0131-j-C-E-a-a-a-characters-l-_00a1-aa-oe-_002c-error_002d_002d_003e--_2026-_002e_002e_002e-no-brace-commands-_0040--_002e-----_007d-signs-_002d_002d-_002d_002d_002d-_0060_0060-_0027_0027-_0021_005f_0022_0023_0024_0025_0026_0027_0028_0029_002a_002b_002d_002e-_002f_003b_003c_003d_003e_003f_005b_005c_005d_005e_005f_0060_007c_007e',
   'transliterated complex line no unidecode');
 
-my $top_text = ' tOp';
-my $top_tree = $parser->parse_texi_text($top_text);
+my $top_no_space = 'tOp';
+my $top_tree = $parser->parse_texi_line($top_no_space);
 my $top_normalized = normalize_node($top_tree);
 is ($top_normalized, 'Top', 'normalize Top node');
 
+my $top_and_space_before = ' tOp';
+# when parsed with parse_texi_text, the text is put in a paragraph
+# and spaces before the text is put in a speicial content for
+# spaces before paragraphs, that are ignored afterwards
+my $top_and_space_before_tree_text = 
$parser->parse_texi_text($top_and_space_before);
+my $top_and_space_before_text_normalized
+   = normalize_node($top_and_space_before_tree_text);
+is ($top_and_space_before_text_normalized, 'Top',
+    'normalize Top node preceded by space as text');
+#print STDERR Data::Dumper->Dump([$top_and_space_before_tree_text]);
+
+# when parsed with parse_texi_line, the text is not put in a
+# paragraph and the first space is retained, such that there
+# is no normalization
+my $top_and_space_before_tree_line = 
$parser->parse_texi_line($top_and_space_before);
+my $top_and_space_before_line_normalized
+   = normalize_node($top_and_space_before_tree_line);
+is ($top_and_space_before_line_normalized, '-tOp',
+    'normalize Top node preceded by space as line');
+#print STDERR Data::Dumper->Dump([$top_and_space_before_tree_line]);
+
 my $top_and_spaces_text = 'TOP ';
-my $top_and_spaces_tree = $parser->parse_texi_text($top_and_spaces_text);
+my $top_and_spaces_tree = $parser->parse_texi_line($top_and_spaces_text);
 my $top_and_spaces_normalized = normalize_node($top_and_spaces_tree);
 is ($top_and_spaces_normalized, 'TOP-', 'normalize Top node followed by 
spaces');
diff --git a/tp/t/protect_character_in_texinfo.t 
b/tp/t/protect_character_in_texinfo.t
index 05804ddc01..08cbb029e6 100644
--- a/tp/t/protect_character_in_texinfo.t
+++ b/tp/t/protect_character_in_texinfo.t
@@ -5,9 +5,9 @@ use Texinfo::ModulePath (undef, undef, 'updirs' => 2);
 
 use Test::More;
 
-BEGIN { plan tests => 4; }
+BEGIN { plan tests => 7; }
 
-use Texinfo::Parser qw(parse_texi_text);
+use Texinfo::Parser qw(parse_texi_line parse_texi_text);
 use Texinfo::Common qw(protect_comma_in_tree protect_colon_in_tree
       protect_node_after_label_in_tree);
 use Texinfo::Convert::Texinfo;
@@ -21,37 +21,55 @@ sub run_test($$$$)
   my $out = shift;
   my $name = shift;
 
-  my $tree = parse_texi_text(undef, $in);
-  if ($do->{'protect_comma'}) {
-    $tree = protect_comma_in_tree($tree);
+  my ($reference_as_text, $reference_as_line);
+  if ($out) {
+    ($reference_as_text, $reference_as_line) = @$out;
+    $reference_as_line = $reference_as_text if not defined($reference_as_line);
   }
-  if ($do->{'protect_colon'}) {
-    $tree = protect_colon_in_tree($tree);
-  }
-  if ($do->{'protect_node_after_label'}) {
-    $tree = protect_node_after_label_in_tree($tree);
+
+  my $tree_as_text = parse_texi_text(undef, $in);
+  my $tree_as_line = parse_texi_line(undef, $in);
+
+  foreach my $tree ($tree_as_text, $tree_as_line) {
+    if ($do->{'protect_comma'}) {
+      $tree = protect_comma_in_tree($tree);
+    }
+    if ($do->{'protect_colon'}) {
+      $tree = protect_colon_in_tree($tree);
+    }
+    if ($do->{'protect_node_after_label'}) {
+      $tree = protect_node_after_label_in_tree($tree);
+    }
   }
-  
-  my $texi_result = Texinfo::Convert::Texinfo::convert_to_texinfo($tree);
+  my $texi_result_as_text
+     = Texinfo::Convert::Texinfo::convert_to_texinfo($tree_as_text);
+  my $texi_result_as_line
+     = Texinfo::Convert::Texinfo::convert_to_texinfo($tree_as_line);
 
   if (!defined($out)) {
-    print STDERR " --> $name:\n$texi_result";
+    print STDERR " --> $name:\n$texi_result_as_text\n$texi_result_as_line";
   } else {
-    is ($texi_result, $out, $name);
+    is ($texi_result_as_text, $reference_as_text, "text $name");
+    is ($texi_result_as_line, $reference_as_line, "line $name");
   }
 }
 
 run_test({'protect_comma' => 1},
-'Some, text,,,@code{,} @asis{, text} @verb{:v,:} @,c', 
-'Some@comma{} text@comma{}@comma{}@comma{}@code{@comma{}} @asis{@comma{} text} 
@verb{:v,:} @,c', 
+'Some, text,,,@code{,} @asis{, text} @verb{:v,:} @,c',
+['Some@comma{} text@comma{}@comma{}@comma{}@code{@comma{}} @asis{@comma{} 
text} @verb{:v,:} @,c',
+ undef],
 'protect comma');
 
 run_test({'protect_colon' => 1},
 'Some :: colons: @code{:} @verb{: in verb::} @:.:',
-'Some @asis{::} colons@asis{:} @code{@asis{:}} @verb{: in verb::} @:.@asis{:}',
+['Some @asis{::} colons@asis{:} @code{@asis{:}} @verb{: in verb::} 
@:.@asis{:}',
+ undef],
 'protect colon');
 
 run_test({'protect_node_after_label' => 1},
 "\t\t".'., text @code{,.t.} @verb{:, .:} .'."\t t",
-'@asis{                }@asis{.,} text @code{@asis{,.}t@asis{.}} @verb{:, .:} 
@asis{.  } t',
+# the result when parsed as text is different as the first two spaces are
+# before the first paragraph, while there is no paragraph when parsed as line
+['@asis{               }@asis{.,} text @code{@asis{,.}t@asis{.}} @verb{:, .:} 
@asis{.  } t',
+ '@asis{               .,} text @code{@asis{,.}t@asis{.}} @verb{:, .:} @asis{. 
} t'],
 'protect node after label characters');
diff --git a/tp/t/test_brace_count.t b/tp/t/test_brace_count.t
index 645abc5cfb..a335c7ad1d 100644
--- a/tp/t/test_brace_count.t
+++ b/tp/t/test_brace_count.t
@@ -5,20 +5,20 @@ use Texinfo::ModulePath (undef, undef, 'updirs' => 2);
 
 use Test::More;
 
-BEGIN { plan tests => 7; }
+BEGIN { plan tests => 13; }
 
-use Texinfo::Parser qw(parse_texi_text);
+use Texinfo::Parser qw(parse_texi_line);
 use Texinfo::Common;
 
 ok(1, "modules loading");
 
-my ($before, $after, $braces_count) 
+my ($before, $after, $braces_count)
   = Texinfo::Common::_find_end_brace(" aa (bbb ()ccc)(g))j (gg", 1);
 is ($before, " aa (bbb ()ccc)(g))", "before with brace closed");
 is ($after, "j (gg", "after with brace closed");
 ok ($braces_count == 0, "braces count 0 with brace closed");
 my $string_no_close = " kjsdf ( k)lj(";
-($before, $after, $braces_count) 
+($before, $after, $braces_count)
   = Texinfo::Common::_find_end_brace($string_no_close, 2);
 ok (($braces_count == 3 and $before eq $string_no_close and !defined($after)),
     "more braces opened");
@@ -30,10 +30,27 @@ ok (($braces_count == 1 and $before eq $string_no_brace and 
!defined($after)),
 my $string_open_brace_and_text = " (other ";
 ($before, $after, $braces_count)
   = Texinfo::Common::_find_end_brace($string_open_brace_and_text, 1);
-ok (($braces_count == 2 and $before eq $string_open_brace_and_text 
+ok (($braces_count == 2 and $before eq $string_open_brace_and_text
      and !defined($after)), "more braces opened and text");
 
 
+sub _find_text($)
+{
+  my $root = shift;
+  my $current = $root;
+  while ($current->{'type'} and ($current->{'type'} eq 'root_line'
+                           # following could be useful if parse_texi_text is 
used
+                                 or $current->{'type'} eq 'before_node_section'
+                                 or $current->{'type'} eq 'document_root'
+                                 or $current->{'type'} eq 'paragraph')) {
+    $current = $current->{'contents'}->[0];
+  }
+  if (not exists($current->{'text'})) {
+    return undef;
+  }
+  return $current;
+}
+
 sub run_test($$$$)
 {
   my $in = shift;
@@ -41,10 +58,11 @@ sub run_test($$$$)
   my $ref_braces_count = shift;
   my $name = shift;
 
-  my $tree = parse_texi_text(undef, $in);
+  my $tree = parse_texi_line(undef, $in);
+  my $text_element = _find_text($tree);
 
-  my $braces_count 
-    = Texinfo::Common::_count_opened_tree_braces($tree, $initial_brace_count);
+  my $braces_count
+    = Texinfo::Common::_count_opened_tree_braces($text_element, 
$initial_brace_count);
   if (!defined($ref_braces_count)) {
     print STDERR " --> $name ($in): $braces_count\n";
   } else {
@@ -52,8 +70,16 @@ sub run_test($$$$)
   }
 }
 
+run_test(')', 1, 0, 'brace closed');
+run_test('(dir)', 0, 0, 'text in brace');
+run_test('ab(sdffsd) other(gg))', 1, 0, 'one initial all closed');
+run_test('(sdffsd)) aaa', 1, 0, 'too much braces');
+run_test('aaa) )   ( ', 2, 0, 'more reopened');
+run_test(' aaa)   ( (', 2, 3, 'still open');
+
 # Note: these tests are disabled because the code doesn't look at
-# parentheses nested inside commands anymore.
+# parentheses nested inside commands.  The run_test() functions would
+# have to be modified to handle tree with @-commands
 
 # run_test('@code{(sdffsd)} other @code{(gg} ))', 1, 0, 'brace in code');
 # run_test('@code{(sdffsd)) aaa}', 1, 0, 'too much braces');
diff --git a/tp/t/test_is_content_empty.t b/tp/t/test_is_content_empty.t
index 63ad95ce6f..4fe60062a1 100644
--- a/tp/t/test_is_content_empty.t
+++ b/tp/t/test_is_content_empty.t
@@ -5,9 +5,9 @@ use Texinfo::ModulePath (undef, undef, 'updirs' => 2);
 
 use Test::More;
 
-BEGIN { plan tests => 12; }
+BEGIN { plan tests => 23; }
 
-use Texinfo::Parser qw(parse_texi_text);
+use Texinfo::Parser qw(parse_texi_text parse_texi_line);
 #use Texinfo::Convert::Texinfo;
 use Texinfo::Common;
 use Data::Dumper;
@@ -20,12 +20,17 @@ sub test_is_empty($$$;$)
   my $is_empty = shift;
   my $in = shift;
   my $do_not_ignore_index_entries = shift;
-  my $tree = parse_texi_text(undef, $in);
-  my $result = Texinfo::Common::is_content_empty($tree, 
$do_not_ignore_index_entries);
+  my $tree_as_text = parse_texi_text(undef, $in);
+  my $tree_as_line = parse_texi_line(undef, $in);
+  my $result_as_text = Texinfo::Common::is_content_empty($tree_as_text,
+                                               $do_not_ignore_index_entries);
+  my $result_as_line = Texinfo::Common::is_content_empty($tree_as_line,
+                                               $do_not_ignore_index_entries);
   if (not defined($is_empty)) {
-    print STDERR " --> $name: $result\n";
+    print STDERR " --> $name: $result_as_text, $result_as_line\n";
   } else {
-    is($result, $is_empty, $name);
+    is($result_as_text, $is_empty, "text $name");
+    is($result_as_line, $is_empty, "line $name");
   }
 }
 
diff --git a/tp/t/test_utils.pl b/tp/t/test_utils.pl
index 1fe8301f81..e3cc93ecb6 100644
--- a/tp/t/test_utils.pl
+++ b/tp/t/test_utils.pl
@@ -880,7 +880,7 @@ sub test($$)
   my $result;
   if (!$test_file) {
     print STDERR "  TEST $test_name\n" if ($self->{'DEBUG'});
-    $result = $parser->parse_texi_text($test_text, 1);
+    $result = $parser->parse_texi_text($test_text);
     if (defined($test_input_file_name)) {
       $parser->{'info'}->{'input_file_name'} = $test_input_file_name;
     }