Subject: Real: fix heap buffer overflow (CVE-2010-3907)
 Malformatted files may have a zero i_subpackets value. In this case,
 we cannot use the array, but we still have to free it (calloc(0)).
Author: Rémi Denis-Courmont <remi@remlab.net>
Bug-Ubuntu: https://launchpad.net/bugs/690173
Origin: upstream, http://git.videolan.org/gitweb.cgi?p=vlc/vlc-1.1.git;a=commit;h=5264082844c1deb05585c245525fd55f9a9cab41

diff --git a/modules/demux/real.c b/modules/demux/real.c
index dee5b52..e3b6a07 100644
--- a/modules/demux/real.c
+++ b/modules/demux/real.c
@@ -252,11 +252,8 @@ static void Close( vlc_object_t *p_this )
             if( tk->p_subpackets[ j ] )
                 block_Release( tk->p_subpackets[ j ] );
         }
-        if( tk->i_subpackets )
-        {
-            free( tk->p_subpackets );
-            free( tk->p_subpackets_timecode );
-        }
+        free( tk->p_subpackets );
+        free( tk->p_subpackets_timecode );
         if( tk->p_sipr_packet )
             block_Release( tk->p_sipr_packet );
         free( tk );
@@ -637,6 +634,11 @@ static void DemuxAudioMethod1( demux_t *p_demux, real_track_t *tk, mtime_t i_pts
 
         for( int i = 0; i < i_num; i++ )
         {
+            int i_index = tk->i_subpacket_h * i +
+                          ((tk->i_subpacket_h + 1) / 2) * (y&1) + (y>>1);
+            if( i_index >= tk->i_subpackets )
+                return;
+
             block_t *p_block = block_New( p_demux, tk->i_subpacket_size );
             if( !p_block )
                 return;
@@ -649,9 +651,6 @@ static void DemuxAudioMethod1( demux_t *p_demux, real_track_t *tk, mtime_t i_pts
 
             p_buf += tk->i_subpacket_size;
 
-            int i_index = tk->i_subpacket_h * i +
-                          ((tk->i_subpacket_h + 1) / 2) * (y&1) + (y>>1);
-
             if( tk->p_subpackets[i_index] != NULL )
             {
                 msg_Dbg(p_demux, "p_subpackets[ %d ] not null!",  i_index );
@@ -671,14 +670,16 @@ static void DemuxAudioMethod1( demux_t *p_demux, real_track_t *tk, mtime_t i_pts
 
         for( int i = 0; i < tk->i_subpacket_h / 2; i++ )
         {
+            int i_index = (i * 2 * tk->i_frame_size / tk->i_coded_frame_size) + y;
+            if( i_index >= tk->i_subpackets )
+                return;
+
             block_t *p_block = block_New( p_demux, tk->i_coded_frame_size);
             if( !p_block )
                 return;
             if( &p_buf[tk->i_coded_frame_size] > &p_sys->buffer[p_sys->i_buffer] )
                 return;
 
-            int i_index = (i * 2 * tk->i_frame_size / tk->i_coded_frame_size) + y;
-
             memcpy( p_block->p_buffer, p_buf, tk->i_coded_frame_size );
             p_block->i_dts =
             p_block->i_pts = i_index == 0 ? i_pts : VLC_TS_INVALID;