flac/flac-openssl.patch


Sender: =?UTF-8?Q?Cristian_Rodr=C3=ADguez?= <cr@cristianrodriguez.net>
From: =?UTF-8?q?Cristian=20Rodr=C3=ADguez?= <crrodriguez@opensuse.org>
To: flac-dev@xiph.org
Cc: =?UTF-8?q?Cristian=20Rodr=C3=ADguez?= <crrodriguez@opensuse.org>
Subject: =?UTF-8?q?=5BPATCH=5D=20Optionally=2C=20allow=20distros=20to=20use=20openssl=20for=20MD5=20verification?=
Date: Sat,  5 May 2012 17:34:31 -0400
Message-Id: <1336253671-7262-1-git-send-email-crrodriguez@opensuse.org>
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This has the advantage of being more efficient than the included
routines and allows distros to centralize crypto mainteniance on
a few libraries.
---
 configure.ac                      |    4 +-
 m4/ax_check_openssl.m4            |  124 +++++++++++++++++++++++++++++++++++++
 src/libFLAC/Makefile.am           |    2 +-
 src/libFLAC/include/private/md5.h |    8 ++-
 src/libFLAC/md5.c                 |   38 +++++++++++
 src/libFLAC/stream_decoder.c      |   30 +++++++--
 src/libFLAC/stream_encoder.c      |   30 +++++++--
 7 files changed, 220 insertions(+), 16 deletions(-)
 create mode 100644 m4/ax_check_openssl.m4

--- flac-1.2.1_git201212051942.orig/configure.ac
+++ flac-1.2.1_git201212051942/configure.ac
@@ -58,7 +58,7 @@ AM_PROG_CC_C_O
 AC_C_INLINE
 AC_C_VARARRAYS
 AC_C_TYPEOF
-
+AC_FUNC_ALLOCA
 AC_CHECK_HEADERS(stdint.h)
 AC_SUBST(HAVE_STDINT_H)
 AC_CHECK_HEADERS(inttypes.h)
@@ -85,6 +85,8 @@ dnl check for getopt in standard library
 dnl AC_CHECK_FUNCS(getopt_long , , [LIBOBJS="$LIBOBJS getopt.o getopt1.o"] )
 AC_CHECK_FUNCS(getopt_long, [], [])

+AX_CHECK_OPENSSL([AC_DEFINE([HAVE_OPENSSL], [1], [We have openSSL])])
+
 case "$host_cpu" in
 	i*86)
 		cpu_ia32=true
--- /dev/null
+++ flac-1.2.1_git201212051942/m4/ax_check_openssl.m4
@@ -0,0 +1,124 @@
+# ===========================================================================
+#     http://www.gnu.org/software/autoconf-archive/ax_check_openssl.html
+# ===========================================================================
+#
+# SYNOPSIS
+#
+#   AX_CHECK_OPENSSL([action-if-found[, action-if-not-found]])
+#
+# DESCRIPTION
+#
+#   Look for OpenSSL in a number of default spots, or in a user-selected
+#   spot (via --with-openssl).  Sets
+#
+#     OPENSSL_INCLUDES to the include directives required
+#     OPENSSL_LIBS to the -l directives required
+#     OPENSSL_LDFLAGS to the -L or -R flags required
+#
+#   and calls ACTION-IF-FOUND or ACTION-IF-NOT-FOUND appropriately
+#
+#   This macro sets OPENSSL_INCLUDES such that source files should use the
+#   openssl/ directory in include directives:
+#
+#     #include <openssl/hmac.h>
+#
+# LICENSE
+#
+#   Copyright (c) 2009,2010 Zmanda Inc. <http://www.zmanda.com/>
+#   Copyright (c) 2009,2010 Dustin J. Mitchell <dustin@zmanda.com>
+#
+#   Copying and distribution of this file, with or without modification, are
+#   permitted in any medium without royalty provided the copyright notice
+#   and this notice are preserved. This file is offered as-is, without any
+#   warranty.
+
+#serial 8
+
+AU_ALIAS([CHECK_SSL], [AX_CHECK_OPENSSL])
+AC_DEFUN([AX_CHECK_OPENSSL], [
+    found=false
+    AC_ARG_WITH([openssl],
+        [AS_HELP_STRING([--with-openssl=DIR],
+            [root of the OpenSSL directory])],
+        [
+            case "$withval" in
+            "" | y | ye | yes | n | no)
+            AC_MSG_ERROR([Invalid --with-openssl value])
+              ;;
+            *) ssldirs="$withval"
+              ;;
+            esac
+        ], [
+            # if pkg-config is installed and openssl has installed a .pc file,
+            # then use that information and don't search ssldirs
+            AC_PATH_PROG([PKG_CONFIG], [pkg-config])
+            if test x"$PKG_CONFIG" != x""; then
+                OPENSSL_LDFLAGS=`$PKG_CONFIG openssl --libs-only-L 2>/dev/null`
+                if test $? = 0; then
+                    OPENSSL_LIBS=`$PKG_CONFIG openssl --libs-only-l 2>/dev/null`
+                    OPENSSL_INCLUDES=`$PKG_CONFIG openssl --cflags-only-I 2>/dev/null`
+                    found=true
+                fi
+            fi
+
+            # no such luck; use some default ssldirs
+            if ! $found; then
+                ssldirs="/usr/local/ssl /usr/lib/ssl /usr/ssl /usr/pkg /usr/local /usr"
+            fi
+        ]
+        )
+
+
+    # note that we #include <openssl/foo.h>, so the OpenSSL headers have to be in
+    # an 'openssl' subdirectory
+
+    if ! $found; then
+        OPENSSL_INCLUDES=
+        for ssldir in $ssldirs; do
+            AC_MSG_CHECKING([for openssl/ssl.h in $ssldir])
+            if test -f "$ssldir/include/openssl/ssl.h"; then
+                OPENSSL_INCLUDES="-I$ssldir/include"
+                OPENSSL_LDFLAGS="-L$ssldir/lib"
+                OPENSSL_LIBS="-lssl -lcrypto"
+                found=true
+                AC_MSG_RESULT([yes])
+                break
+            else
+                AC_MSG_RESULT([no])
+            fi
+        done
+
+        # if the file wasn't found, well, go ahead and try the link anyway -- maybe
+        # it will just work!
+    fi
+
+    # try the preprocessor and linker with our new flags,
+    # being careful not to pollute the global LIBS, LDFLAGS, and CPPFLAGS
+
+    AC_MSG_CHECKING([whether compiling and linking against OpenSSL works])
+    echo "Trying link with OPENSSL_LDFLAGS=$OPENSSL_LDFLAGS;" \
+        "OPENSSL_LIBS=$OPENSSL_LIBS; OPENSSL_INCLUDES=$OPENSSL_INCLUDES" >&AS_MESSAGE_LOG_FD
+
+    save_LIBS="$LIBS"
+    save_LDFLAGS="$LDFLAGS"
+    save_CPPFLAGS="$CPPFLAGS"
+    LDFLAGS="$LDFLAGS $OPENSSL_LDFLAGS"
+    LIBS="$OPENSSL_LIBS $LIBS"
+    CPPFLAGS="$OPENSSL_INCLUDES $CPPFLAGS"
+    AC_LINK_IFELSE(
+        [AC_LANG_PROGRAM([#include <openssl/ssl.h>], [SSL_new(NULL)])],
+        [
+            AC_MSG_RESULT([yes])
+            $1
+        ], [
+            AC_MSG_RESULT([no])
+            $2
+        ])
+    CPPFLAGS="$save_CPPFLAGS"
+    LDFLAGS="$save_LDFLAGS"
+    LIBS="$save_LIBS"
+
+    AC_SUBST([OPENSSL_INCLUDES])
+    AC_SUBST([OPENSSL_LIBS])
+    AC_SUBST([OPENSSL_LDFLAGS])
+])
--- flac-1.2.1_git201212051942.orig/src/libFLAC/Makefile.am
+++ flac-1.2.1_git201212051942/src/libFLAC/Makefile.am
@@ -79,7 +79,7 @@ endif
 endif
 endif

-libFLAC_la_LIBADD = $(LOCAL_EXTRA_LIBADD) @OGG_LIBS@ -lm
+libFLAC_la_LIBADD = $(LOCAL_EXTRA_LIBADD) @OPENSSL_LIBS@ @OGG_LIBS@ -lm

 SUBDIRS = $(ARCH_SUBDIRS) include .

@@ -131,5 +131,5 @@ libFLAC_sources = \
 libFLAC_la_SOURCES = $(libFLAC_sources)

 # needed for test_libFLAC
-libFLAC_static_la_LIBADD = $(LOCAL_EXTRA_LIBADD)
+libFLAC_static_la_LIBADD = $(LOCAL_EXTRA_LIBADD) $(OPENSSL_LIBS)
 libFLAC_static_la_SOURCES = $(libFLAC_sources)
--- flac-1.2.1_git201212051942.orig/src/libFLAC/include/private/md5.h
+++ flac-1.2.1_git201212051942/src/libFLAC/include/private/md5.h
@@ -28,6 +28,11 @@

 #include "FLAC/ordinals.h"

+#if defined(HAVE_OPENSSL)
+#include <openssl/evp.h>
+#define FLAC__MD5Context EVP_MD_CTX
+#else
+#define EVP_MAX_MD_SIZE 16
 typedef struct {
 	FLAC__uint32 in[16];
 	FLAC__uint32 buf[4];
@@ -37,7 +42,8 @@ typedef struct {
 } FLAC__MD5Context;

 void FLAC__MD5Init(FLAC__MD5Context *context);
-void FLAC__MD5Final(FLAC__byte digest[16], FLAC__MD5Context *context);
+void FLAC__MD5Final(FLAC__byte digest[EVP_MAX_MD_SIZE], FLAC__MD5Context *context);
+#endif

 FLAC__bool FLAC__MD5Accumulate(FLAC__MD5Context *ctx, const FLAC__int32 * const signal[], unsigned channels, unsigned samples, unsigned bytes_per_sample);

--- flac-1.2.1_git201212051942.orig/src/libFLAC/md5.c
+++ flac-1.2.1_git201212051942/src/libFLAC/md5.c
@@ -5,6 +5,19 @@
 #include <stdlib.h>		/* for malloc() */
 #include <string.h>		/* for memcpy() */

+#ifdef HAVE_ALLOCA_H
+# include <alloca.h>
+#elif !defined alloca
+# ifdef __GNUC__
+#  define alloca __builtin_alloca
+# elif defined _AIX
+#  define alloca __alloca
+# elif defined _MSC_VER
+#  include <malloc.h>
+#  define alloca _alloca
+# endif
+#endif
+
 #include "private/md5.h"
 #include "share/alloc.h"

@@ -35,6 +48,7 @@

 /* The four core functions - F1 is optimized somewhat */

+#if !defined(HAVE_OPENSSL)
 /* #define F1(x, y, z) (x & y | ~x & z) */
 #define F1(x, y, z) (z ^ (x & (y ^ z)))
 #define F2(x, y, z) F1(z, x, y)
@@ -267,6 +281,8 @@ void FLAC__MD5Final(FLAC__byte digest[16
 	memset(ctx, 0, sizeof(*ctx));	/* In case it's sensitive */
 }

+#endif /* !defined(HAVE_OPENSSL) */
+
 /*
  * Convert the incoming audio signal to a byte stream
  */
@@ -401,6 +417,26 @@ FLAC__bool FLAC__MD5Accumulate(FLAC__MD5
 	if((size_t)channels * (size_t)bytes_per_sample > SIZE_MAX / (size_t)samples)
 		return false;

+#if defined(HAVE_OPENSSL)
+    /* Use stack for the most common cases, heap when bytes_needed is larger than 4032 (unlikely)
+     * Note that this is a _very_ conservative estimation.
+     */
+#if defined(_MSC_VER)
+/* see http://msdn.microsoft.com/en-us/library/5471dc8s(v=vs.80).aspx for the rationale */
+    FLAC__byte *tmp = _malloca(bytes_needed);
+#else
+    const FLAC__bool usealloca = (bytes_needed < 4032);
+    FLAC__byte *tmp = usealloca ? alloca(bytes_needed) : safe_malloc_(bytes_needed);
+#endif
+    format_input_(tmp, signal, channels, samples, bytes_per_sample);
+    const FLAC__bool retval = (EVP_DigestUpdate(ctx, tmp , bytes_needed) == 1);
+#if defined(_MSC_VER)
+     _freea(tmp)
+#else
+    if(!usealloca) free(tmp);
+#endif
+    return retval;
+#else
 	if(ctx->capacity < bytes_needed) {
 		FLAC__byte *tmp = realloc(ctx->internal_buf, bytes_needed);
 		if(0 == tmp) {
@@ -418,4 +454,6 @@ FLAC__bool FLAC__MD5Accumulate(FLAC__MD5
 	FLAC__MD5Update(ctx, ctx->internal_buf, bytes_needed);

 	return true;
+#endif
+
 }
--- flac-1.2.1_git201212051942.orig/src/libFLAC/stream_decoder.c
+++ flac-1.2.1_git201212051942/src/libFLAC/stream_decoder.c
@@ -164,7 +164,7 @@ typedef struct FLAC__StreamDecoderPrivat
 	FLAC__bool internal_reset_hack; /* used only during init() so we can call reset to set up the decoder without rewinding the input */
 	FLAC__bool is_seeking;
 	FLAC__MD5Context md5context;
-	FLAC__byte computed_md5sum[16]; /* this is the sum we computed from the decoded data */
+	FLAC__byte computed_md5sum[EVP_MAX_MD_SIZE]; /* this is the sum we computed from the decoded data */
 	/* (the rest of these are only used for seeking) */
 	FLAC__Frame last_frame; /* holds the info of the last frame we seeked to */
 	FLAC__uint64 first_frame_offset; /* hint to the seek routine of where in the stream the first audio frame starts */
@@ -301,7 +301,9 @@ FLAC_API FLAC__StreamDecoder *FLAC__stre
 	decoder->private_->file = 0;

 	set_defaults_(decoder);
-
+#if defined(HAVE_OPENSSL)
+    EVP_MD_CTX_init(&decoder->private_->md5context);
+#endif
 	decoder->protected_->state = FLAC__STREAM_DECODER_UNINITIALIZED;

 	return decoder;
@@ -327,7 +329,9 @@ FLAC_API void FLAC__stream_decoder_delet

 	for(i = 0; i < FLAC__MAX_CHANNELS; i++)
 		FLAC__format_entropy_coding_method_partitioned_rice_contents_clear(&decoder->private_->partitioned_rice_contents[i]);
-
+#if defined(HAVE_OPENSSL)
+    EVP_MD_CTX_cleanup(&decoder->private_->md5context);
+#endif
 	free(decoder->private_);
 	free(decoder->protected_);
 	free(decoder);
@@ -647,8 +651,16 @@ FLAC_API FLAC__bool FLAC__stream_decoder
 	/* see the comment in FLAC__seekable_stream_decoder_reset() as to why we
 	 * always call FLAC__MD5Final()
 	 */
+#if defined(HAVE_OPENSSL)
+    /* decoder->private_->computed_md5sum is NULL when decoder->private_->do_md5_checking == false
+    * that causes assertion failure crash in openSSL.
+    */
+    if(decoder->private_->do_md5_checking) {
+        md5_failed = (EVP_DigestFinal_ex(&decoder->private_->md5context, decoder->private_->computed_md5sum, NULL) == 0);
+    }
+#else
 	FLAC__MD5Final(decoder->private_->computed_md5sum, &decoder->private_->md5context);
-
+#endif
 	if(decoder->private_->has_seek_table && 0 != decoder->private_->seek_table.data.seek_table.points) {
 		free(decoder->private_->seek_table.data.seek_table.points);
 		decoder->private_->seek_table.data.seek_table.points = 0;
@@ -999,11 +1011,15 @@ FLAC_API FLAC__bool FLAC__stream_decoder
 	 * FLAC__stream_decoder_finish() to make sure things are always cleaned up
 	 * properly.
 	 */
-	FLAC__MD5Init(&decoder->private_->md5context);
+    decoder->private_->first_frame_offset = 0;

-	decoder->private_->first_frame_offset = 0;
-	decoder->private_->unparseable_frame_count = 0;
+    decoder->private_->unparseable_frame_count = 0;

+#if defined(HAVE_OPENSSL)
+    return (EVP_DigestInit_ex(&decoder->private_->md5context, EVP_md5(), NULL) == 1);
+#else
+▷⋅⋅⋅FLAC__MD5Init(&decoder->private_->md5context);
+#endif
 	return true;
 }

--- flac-1.2.1_git201212051942.orig/src/libFLAC/stream_encoder.c
+++ flac-1.2.1_git201212051942/src/libFLAC/stream_encoder.c
@@ -562,7 +562,9 @@ FLAC_API FLAC__StreamEncoder *FLAC__stre
 		FLAC__format_entropy_coding_method_partitioned_rice_contents_init(&encoder->private_->partitioned_rice_contents_extra[i]);

 	encoder->protected_->state = FLAC__STREAM_ENCODER_UNINITIALIZED;
-
+#if defined(HAVE_OPENSSL)
+    if(encoder->protected_->do_md5) EVP_MD_CTX_init(&encoder->private_->md5context);
+#endif
 	return encoder;
 }

@@ -596,6 +598,11 @@ FLAC_API void FLAC__stream_encoder_delet
 		FLAC__format_entropy_coding_method_partitioned_rice_contents_clear(&encoder->private_->partitioned_rice_contents_extra[i]);

 	FLAC__bitwriter_delete(encoder->private_->frame);
+#if defined(HAVE_OPENSSL)
+    if(encoder->protected_->do_md5) {
+        EVP_MD_CTX_cleanup(&encoder->private_->md5context);
+    }
+#endif
 	free(encoder->private_);
 	free(encoder->protected_);
 	free(encoder);
@@ -1029,8 +1036,15 @@ static FLAC__StreamEncoderInitStatus ini
 	encoder->private_->streaminfo.data.stream_info.bits_per_sample = encoder->protected_->bits_per_sample;
 	encoder->private_->streaminfo.data.stream_info.total_samples = encoder->protected_->total_samples_estimate; /* we will replace this later with the real total */
 	memset(encoder->private_->streaminfo.data.stream_info.md5sum, 0, 16); /* we don't know this yet; have to fill it in later */
-	if(encoder->protected_->do_md5)
-		FLAC__MD5Init(&encoder->private_->md5context);
+	if(encoder->protected_->do_md5) {
+#if defined(HAVE_OPENSSL)
+        if(EVP_DigestInit_ex(&encoder->private_->md5context, EVP_md5(), NULL) == 0) {
+            return FLAC__STREAM_ENCODER_INIT_STATUS_ENCODER_ERROR;
+        }
+#else
+        FLAC__MD5Init(&encoder->private_->md5context);
+#endif
+    }
 	if(!FLAC__add_metadata_block(&encoder->private_->streaminfo, encoder->private_->frame)) {
 		encoder->protected_->state = FLAC__STREAM_ENCODER_FRAMING_ERROR;
 		return FLAC__STREAM_ENCODER_INIT_STATUS_ENCODER_ERROR;
@@ -1299,9 +1313,13 @@ FLAC_API FLAC__bool FLAC__stream_encoder
 		}
 	}

-	if(encoder->protected_->do_md5)
-		FLAC__MD5Final(encoder->private_->streaminfo.data.stream_info.md5sum, &encoder->private_->md5context);
-
+	if(encoder->protected_->do_md5) {
+#if defined(HAVE_OPENSSL)
+        error = (EVP_DigestFinal_ex(&encoder->private_->md5context, encoder->private_->streaminfo.data.stream_info.md5sum, NULL) == 0);
+#else
+        FLAC__MD5Final(encoder->private_->streaminfo.data.stream_info.md5sum, &encoder->private_->md5context);
+#endif
+    }
 	if(!encoder->private_->is_being_deleted) {
 		if(encoder->protected_->state == FLAC__STREAM_ENCODER_OK) {
 			if(encoder->private_->seek_callback) {