[Devel] FT_Outline_Get_Orientation is very broken

[Anders Kaseorg]

While playing with the path stroker, I discovered that 
FT_Outline_Get_Orientation is returning wrong values. There are several 
obvious problems with the code. I've attached a patch that should fix it.

I haven't checked whether the versions in src/base/ftsynth.c and 
src/autohint/ahglyph.c work. (But should there really be three separate 
functions that do the same thing?)

Anders

Index: src/base/ftoutln.c
===================================================================
RCS file: /cvs/freetype/freetype2/src/base/ftoutln.c,v
retrieving revision 1.57
diff -u -r1.57 ftoutln.c
--- ftoutln.c   2004/01/22 09:07:10     1.57
+++ ftoutln.c   2004/11/22 20:54:11
@@ -663,6 +663,8 @@
   } FT_OrientationExtremumRec;
 
 
+#define FT_ORIENTATION_UNDEF -1
+
   static FT_Orientation
   ft_orientation_extremum_compute( FT_OrientationExtremumRec*  extremum,
                                    FT_Outline*                 outline )
@@ -684,33 +686,36 @@
     {
       prev = ( prev == first ) ? last : prev - 1;
       if ( prev == point )
-        return FT_ORIENTATION_TRUETYPE;  /* degenerate case */
+        return FT_ORIENTATION_UNDEF;  /* degenerate case */
 
-    } while ( prev->x != point->x || prev->y != point->y );
+    } while ( prev->x == point->x && prev->y == point->y );
 
     do
     {
       next = ( next == last ) ? first : next + 1;
       if ( next == point )
-        return FT_ORIENTATION_TRUETYPE;  /* shouldn't happen */
+        return FT_ORIENTATION_UNDEF;  /* shouldn't happen */
 
-    } while ( next->x != point->x || next->y != point->y );
+    } while ( next->x == point->x || next->y == point->y );
+    /* (Yes, this one is ||, not &&.) */
 
     /* now compute the orientation of the `out' vector relative */
     /* to the `in' vector.                                      */
     angle_in  = FT_Atan2( point->x - prev->x,  point->y - prev->y );
     angle_out = FT_Atan2( next->x  - point->x, next->y  - point->y );
 
-    return ( FT_Angle_Diff( angle_in, angle_out ) >= 0 )
-             ? FT_ORIENTATION_TRUETYPE
-             : FT_ORIENTATION_POSTSCRIPT;
+    return ( FT_Angle_Diff( angle_in, angle_out ) > 1e-5 )
+       ? FT_ORIENTATION_POSTSCRIPT
+       : ( ( FT_Angle_Diff( angle_in, angle_out ) < 1e-5 )
+           ? FT_ORIENTATION_TRUETYPE
+           : FT_ORIENTATION_UNDEF );
   }
 
 
   FT_EXPORT_DEF( FT_Orientation )
   FT_Outline_Get_Orientation( FT_Outline*  outline )
   {
-    FT_Orientation  result = FT_ORIENTATION_TRUETYPE;
+    FT_Orientation  result = FT_ORIENTATION_UNDEF;
 
 
     if ( outline && outline->n_points > 0 )
@@ -773,20 +778,23 @@
             }
           }
         }
+      }
 
-        if ( xmin.index >= 0 )
-          result = ft_orientation_extremum_compute( &xmin, outline );
+      if ( result == FT_ORIENTATION_UNDEF && xmin.index >= 0 )
+        result = ft_orientation_extremum_compute( &xmin, outline );
 
-        else if ( xmax.index >= 0 )
-          result = ft_orientation_extremum_compute( &xmax, outline );
+      if ( result == FT_ORIENTATION_UNDEF && xmax.index >= 0 )
+        result = ft_orientation_extremum_compute( &xmax, outline );
 
-        else if ( ymin.index >= 0 )
-          result = ft_orientation_extremum_compute( &ymin, outline );
+      if ( result == FT_ORIENTATION_UNDEF && ymin.index >= 0 )
+        result = ft_orientation_extremum_compute( &ymin, outline );
 
-        else if ( ymax.index >= 0 )
-          result = ft_orientation_extremum_compute( &ymax, outline );
-      }
+      if ( result == FT_ORIENTATION_UNDEF && ymax.index >= 0 )
+        result = ft_orientation_extremum_compute( &ymax, outline );
     }
+
+    if ( result == FT_ORIENTATION_UNDEF )
+      result = FT_ORIENTATION_TRUETYPE;
 
     return result;
   }