implement CryptoAPI RC4+SHA1 encryption scheme for xls import

there might be other variants out there in practice, but this
works for default encrypted xls of excel 2013

Change-Id: I91c0e1d1d95fbd1c68966650e7ac7d23276bcbe3

3 files changed, 170 insertions, 47 deletions

diff --git a/sc/source/filter/excel/xicontent.cxx b/sc/source/filter/excel/xicontent.cxx
index fdd43aef03e6..c854d75af154 100644
--- a/sc/source/filter/excel/xicontent.cxx
+++ b/sc/source/filter/excel/xicontent.cxx
@@ -64,6 +64,7 @@
 #include <memory>
 #include <utility>
 #include <o3tl/make_unique.hxx>
+#include <oox/helper/helper.hxx>
 
 using ::com::sun::star::uno::Sequence;
 using ::std::unique_ptr;
@@ -1110,21 +1111,80 @@ XclImpDecrypterRef lclReadFilepass8_Standard( XclImpStream& rStrm )
     OSL_ENSURE( rStrm.GetRecLeft() == 48, "lclReadFilepass8 - wrong record size" );
     if( rStrm.GetRecLeft() == 48 )
     {
-        sal_uInt8 pnSalt[ 16 ];
-        sal_uInt8 pnVerifier[ 16 ];
-        sal_uInt8 pnVerifierHash[ 16 ];
-        rStrm.Read( pnSalt, 16 );
-        rStrm.Read( pnVerifier, 16 );
-        rStrm.Read( pnVerifierHash, 16 );
-        xDecr.reset( new XclImpBiff8Decrypter( pnSalt, pnVerifier, pnVerifierHash ) );
+        std::vector<sal_uInt8> aSalt(16);
+        std::vector<sal_uInt8> aVerifier(16);
+        std::vector<sal_uInt8> aVerifierHash(16);
+        rStrm.Read(aSalt.data(), 16);
+        rStrm.Read(aVerifier.data(), 16);
+        rStrm.Read(aVerifierHash.data(), 16);
+        xDecr.reset(new XclImpBiff8StdDecrypter(aSalt, aVerifier, aVerifierHash));
     }
     return xDecr;
 }
 
-XclImpDecrypterRef lclReadFilepass8_Strong( XclImpStream& /*rStrm*/ )
+XclImpDecrypterRef lclReadFilepass8_Strong(XclImpStream& rStream)
 {
-    // not supported
-    return XclImpDecrypterRef();
+    //Its possible there are other variants in existance but these
+    //are the defaults I get with Excel 2013
+    XclImpDecrypterRef xDecr;
+
+    msfilter::RC4EncryptionInfo info;
+
+    info.header.flags = rStream.ReaduInt32();
+    if (oox::getFlag( info.header.flags, msfilter::ENCRYPTINFO_EXTERNAL))
+        return xDecr;
+
+    sal_uInt32 nHeaderSize = rStream.ReaduInt32();
+    sal_uInt32 actualHeaderSize = sizeof(info.header);
+
+    if( (nHeaderSize < actualHeaderSize) )
+        return xDecr;
+
+    info.header.flags = rStream.ReaduInt32();
+    info.header.sizeExtra = rStream.ReaduInt32();
+    info.header.algId = rStream.ReaduInt32();
+    info.header.algIdHash = rStream.ReaduInt32();
+    info.header.keyBits = rStream.ReaduInt32();
+    info.header.providedType = rStream.ReaduInt32();
+    info.header.reserved1 = rStream.ReaduInt32();
+    info.header.reserved2 = rStream.ReaduInt32();
+
+    rStream.Ignore(nHeaderSize - actualHeaderSize);
+
+    info.verifier.saltSize = rStream.ReaduInt32();
+    if (info.verifier.saltSize != 16)
+        return xDecr;
+    rStream.Read(&info.verifier.salt, sizeof(info.verifier.salt));
+    rStream.Read(&info.verifier.encryptedVerifier, sizeof(info.verifier.encryptedVerifier));
+
+    info.verifier.encryptedVerifierHashSize = rStream.ReaduInt32();
+    if (info.verifier.encryptedVerifierHashSize != RTL_DIGEST_LENGTH_SHA1)
+        return xDecr;
+    rStream.Read(&info.verifier.encryptedVerifierHash, info.verifier.encryptedVerifierHashSize);
+
+    // check flags and algorithm IDs, required are AES128 and SHA-1
+    if (!oox::getFlag(info.header.flags, msfilter::ENCRYPTINFO_CRYPTOAPI))
+        return xDecr;
+
+    if (oox::getFlag(info.header.flags, msfilter::ENCRYPTINFO_AES))
+        return xDecr;
+
+    if (info.header.algId != msfilter::ENCRYPT_ALGO_RC4)
+        return xDecr;
+
+    // hash algorithm ID 0 defaults to SHA-1 too
+    if (info.header.algIdHash != 0 && info.header.algIdHash != msfilter::ENCRYPT_HASH_SHA1)
+        return xDecr;
+
+    xDecr.reset(new XclImpBiff8CryptoAPIDecrypter(
+        std::vector<sal_uInt8>(info.verifier.salt,
+            info.verifier.salt + SAL_N_ELEMENTS(info.verifier.salt)),
+        std::vector<sal_uInt8>(info.verifier.encryptedVerifier,
+            info.verifier.encryptedVerifier + SAL_N_ELEMENTS(info.verifier.encryptedVerifier)),
+        std::vector<sal_uInt8>(info.verifier.encryptedVerifierHash,
+            info.verifier.encryptedVerifierHash + SAL_N_ELEMENTS(info.verifier.encryptedVerifierHash))));
+
+    return xDecr;
 }
 
 XclImpDecrypterRef lclReadFilepass8( XclImpStream& rStrm )
diff --git a/sc/source/filter/excel/xistream.cxx b/sc/source/filter/excel/xistream.cxx
index 31cf3d5ced08..4f28d9c5101f 100644
--- a/sc/source/filter/excel/xistream.cxx
+++ b/sc/source/filter/excel/xistream.cxx
@@ -192,28 +192,50 @@ sal_uInt16 XclImpBiff5Decrypter::OnRead( SvStream& rStrm, sal_uInt8* pnData, sal
     return nRet;
 }
 
-XclImpBiff8Decrypter::XclImpBiff8Decrypter( sal_uInt8 pnSalt[ 16 ],
-        sal_uInt8 pnVerifier[ 16 ], sal_uInt8 pnVerifierHash[ 16 ] ) :
-    maSalt( pnSalt, pnSalt + 16 ),
-    maVerifier( pnVerifier, pnVerifier + 16 ),
-    maVerifierHash( pnVerifierHash, pnVerifierHash + 16 )
+XclImpBiff8Decrypter::XclImpBiff8Decrypter(const std::vector<sal_uInt8>& rSalt,
+                                           const std::vector<sal_uInt8>& rVerifier,
+                                           const std::vector<sal_uInt8>& rVerifierHash)
+    : maSalt(rSalt)
+    , maVerifier(rVerifier)
+    , maVerifierHash(rVerifierHash)
+    , mpCodec(nullptr)
 {
 }
 
-XclImpBiff8Decrypter::XclImpBiff8Decrypter( const XclImpBiff8Decrypter& rSrc ) :
-    XclImpDecrypter( rSrc ),
-    maEncryptionData( rSrc.maEncryptionData ),
-    maSalt( rSrc.maSalt ),
-    maVerifier( rSrc.maVerifier ),
-    maVerifierHash( rSrc.maVerifierHash )
+XclImpBiff8Decrypter::XclImpBiff8Decrypter(const XclImpBiff8Decrypter& rSrc)
+    : XclImpDecrypter(rSrc)
+    , maEncryptionData(rSrc.maEncryptionData)
+    , maSalt(rSrc.maSalt)
+    , maVerifier(rSrc.maVerifier)
+    , maVerifierHash(rSrc.maVerifierHash)
+    , mpCodec(nullptr)
 {
-    if( IsValid() )
-        maCodec.InitCodec( maEncryptionData );
 }
 
-XclImpBiff8Decrypter* XclImpBiff8Decrypter::OnClone() const
+XclImpBiff8StdDecrypter::XclImpBiff8StdDecrypter(const XclImpBiff8StdDecrypter& rSrc)
+    : XclImpBiff8Decrypter(rSrc)
 {
-    return new XclImpBiff8Decrypter( *this );
+    mpCodec = &maCodec;
+    if (IsValid())
+        maCodec.InitCodec(maEncryptionData);
+}
+
+XclImpBiff8StdDecrypter* XclImpBiff8StdDecrypter::OnClone() const
+{
+    return new XclImpBiff8StdDecrypter(*this);
+}
+
+XclImpBiff8CryptoAPIDecrypter::XclImpBiff8CryptoAPIDecrypter(const XclImpBiff8CryptoAPIDecrypter& rSrc)
+    : XclImpBiff8Decrypter(rSrc)
+{
+    mpCodec = &maCodec;
+    if (IsValid())
+        maCodec.InitCodec(maEncryptionData);
+}
+
+XclImpBiff8CryptoAPIDecrypter* XclImpBiff8CryptoAPIDecrypter::OnClone() const
+{
+    return new XclImpBiff8CryptoAPIDecrypter(*this);
 }
 
 uno::Sequence< beans::NamedValue > XclImpBiff8Decrypter::OnVerifyPassword( const OUString& rPassword )
@@ -232,9 +254,9 @@ uno::Sequence< beans::NamedValue > XclImpBiff8Decrypter::OnVerifyPassword( const
             *aIt = static_cast< sal_uInt16 >( *pcChar );
 
         // init codec
-        maCodec.InitKey( &aPassVect.front(), &maSalt.front() );
-        if ( maCodec.VerifyKey( &maVerifier.front(), &maVerifierHash.front() ) )
-            maEncryptionData = maCodec.GetEncryptionData();
+        mpCodec->InitKey( &aPassVect.front(), &maSalt.front() );
+        if ( mpCodec->VerifyKey( &maVerifier.front(), &maVerifierHash.front() ) )
+            maEncryptionData = mpCodec->GetEncryptionData();
     }
 
     return maEncryptionData;
@@ -247,9 +269,9 @@ bool XclImpBiff8Decrypter::OnVerifyEncryptionData( const uno::Sequence< beans::N
     if( rEncryptionData.getLength() )
     {
         // init codec
-        maCodec.InitCodec( rEncryptionData );
+        mpCodec->InitCodec( rEncryptionData );
 
-        if ( maCodec.VerifyKey( &maVerifier.front(), &maVerifierHash.front() ) )
+        if ( mpCodec->VerifyKey( &maVerifier.front(), &maVerifierHash.front() ) )
             maEncryptionData = rEncryptionData;
     }
 
@@ -269,13 +291,13 @@ void XclImpBiff8Decrypter::OnUpdate( std::size_t nOldStrmPos, std::size_t nNewSt
         /*  Rekey cipher, if block changed or if previous offset in same block. */
         if( (nNewBlock != nOldBlock) || (nNewOffset < nOldOffset) )
         {
-            maCodec.InitCipher( nNewBlock );
+            mpCodec->InitCipher( nNewBlock );
             nOldOffset = 0;     // reset nOldOffset for next if() statement
         }
 
         /*  Seek to correct offset. */
         if( nNewOffset > nOldOffset )
-            maCodec.Skip( nNewOffset - nOldOffset );
+            mpCodec->Skip( nNewOffset - nOldOffset );
     }
 }
 
@@ -293,9 +315,9 @@ sal_uInt16 XclImpBiff8Decrypter::OnRead( SvStream& rStrm, sal_uInt8* pnData, sal
         // read the block from stream
         nRet = nRet + static_cast<sal_uInt16>(rStrm.ReadBytes(pnCurrData, nDecBytes));
         // decode the block inplace
-        maCodec.Decode( pnCurrData, nDecBytes, pnCurrData, nDecBytes );
+        mpCodec->Decode( pnCurrData, nDecBytes, pnCurrData, nDecBytes );
         if( GetOffset( rStrm.Tell() ) == 0 )
-            maCodec.InitCipher( GetBlock( rStrm.Tell() ) );
+            mpCodec->InitCipher( GetBlock( rStrm.Tell() ) );
 
         pnCurrData += nDecBytes;
         nBytesLeft = nBytesLeft - nDecBytes;
diff --git a/sc/source/filter/inc/xistream.hxx b/sc/source/filter/inc/xistream.hxx
index 4e0e63bd84e5..3a53643ef1cf 100644
--- a/sc/source/filter/inc/xistream.hxx
+++ b/sc/source/filter/inc/xistream.hxx
@@ -119,16 +119,7 @@ private:
 /** Decrypts BIFF8 stream contents using the given document identifier. */
 class XclImpBiff8Decrypter : public XclImpDecrypter
 {
-public:
-    explicit            XclImpBiff8Decrypter( sal_uInt8 pnSalt[ 16 ],
-                            sal_uInt8 pnVerifier[ 16 ], sal_uInt8 pnVerifierHash[ 16 ] );
-
 private:
-    /** Private copy c'tor for OnClone(). */
-    explicit            XclImpBiff8Decrypter( const XclImpBiff8Decrypter& rSrc );
-
-    /** Implementation of cloning this object. */
-    virtual XclImpBiff8Decrypter* OnClone() const override;
     /** Implements password verification and initialization of the decoder. */
     virtual css::uno::Sequence< css::beans::NamedValue >
         OnVerifyPassword( const OUString& rPassword ) override;
@@ -143,12 +134,62 @@ private:
     /** Returns the block offset corresponding to the passed stream position. */
     static sal_uInt16   GetOffset( std::size_t nStrmPos );
 
+protected:
+    explicit  XclImpBiff8Decrypter(const std::vector<sal_uInt8>& rSalt,
+                                   const std::vector<sal_uInt8>& rVerifier,
+                                   const std::vector<sal_uInt8>& rVerifierHash);
+
+    explicit  XclImpBiff8Decrypter(const XclImpBiff8Decrypter& rSrc);
+
+    css::uno::Sequence< css::beans::NamedValue > maEncryptionData;
+    std::vector< sal_uInt8 > maSalt;
+    std::vector< sal_uInt8 > maVerifier;
+    std::vector< sal_uInt8 > maVerifierHash;
+    msfilter::MSCodec97* mpCodec;       /// Crypto algorithm implementation.
+};
+
+class XclImpBiff8StdDecrypter : public XclImpBiff8Decrypter
+{
+public:
+    explicit XclImpBiff8StdDecrypter(const std::vector<sal_uInt8>& rSalt,
+                                     const std::vector<sal_uInt8>& rVerifier,
+                                     const std::vector<sal_uInt8>& rVerifierHash)
+        : XclImpBiff8Decrypter(rSalt, rVerifier, rVerifierHash)
+    {
+        mpCodec = &maCodec;
+    }
+
+private:
+    /** Private copy c'tor for OnClone(). */
+    explicit XclImpBiff8StdDecrypter(const XclImpBiff8StdDecrypter& rSrc);
+
+    /** Implementation of cloning this object. */
+    virtual XclImpBiff8StdDecrypter* OnClone() const override;
+
 private:
     ::msfilter::MSCodec_Std97 maCodec;       /// Crypto algorithm implementation.
-    css::uno::Sequence< css::beans::NamedValue > maEncryptionData;
-    ::std::vector< sal_uInt8 > maSalt;
-    ::std::vector< sal_uInt8 > maVerifier;
-    ::std::vector< sal_uInt8 > maVerifierHash;
+};
+
+class XclImpBiff8CryptoAPIDecrypter : public XclImpBiff8Decrypter
+{
+public:
+    explicit XclImpBiff8CryptoAPIDecrypter(const std::vector<sal_uInt8>& rSalt,
+                                           const std::vector<sal_uInt8>& rVerifier,
+                                           const std::vector<sal_uInt8>& rVerifierHash)
+        : XclImpBiff8Decrypter(rSalt, rVerifier, rVerifierHash)
+    {
+        mpCodec = &maCodec;
+    }
+
+private:
+    /** Private copy c'tor for OnClone(). */
+    explicit XclImpBiff8CryptoAPIDecrypter(const XclImpBiff8CryptoAPIDecrypter& rSrc);
+
+    /** Implementation of cloning this object. */
+    virtual XclImpBiff8CryptoAPIDecrypter* OnClone() const override;
+
+private:
+    ::msfilter::MSCodec_CryptoAPI maCodec;       /// Crypto algorithm implementation.
 };
 
 // Stream