diff --git a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/InputDataOperation.java b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/InputDataOperation.java index bb8d6ad73..43fc11b84 100644 --- a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/InputDataOperation.java +++ b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/InputDataOperation.java @@ -53,6 +53,7 @@ import org.sufficientlysecure.keychain.provider.ProviderHelper; import org.sufficientlysecure.keychain.provider.TemporaryFileProvider; import org.sufficientlysecure.keychain.service.InputDataParcel; import org.sufficientlysecure.keychain.service.input.CryptoInputParcel; +import org.sufficientlysecure.keychain.util.CharsetVerifier; /** This operation deals with input data, trying to determine its type as it goes. @@ -67,7 +68,7 @@ import org.sufficientlysecure.keychain.service.input.CryptoInputParcel; */ public class InputDataOperation extends BaseOperation { - final private byte[] buf = new byte[256]; + private final byte[] buf = new byte[256]; public InputDataOperation(Context context, ProviderHelper providerHelper, Progressable progressable) { super(context, providerHelper, progressable); @@ -326,21 +327,37 @@ public class InputDataOperation extends BaseOperation { throw new IOException("Error getting file for writing!"); } + // If this data looks like text, we pipe the incoming data into a charset + // decoder, to see if the data is legal for the assumed charset. + String charset = bd.getCharset(); + CharsetVerifier charsetVerifier = new CharsetVerifier(buf, mimeType, charset); + int totalLength = 0; do { totalLength += len; out.write(buf, 0, len); + charsetVerifier.readBytesFromBuffer(0, len); } while ((len = is.read(buf)) > 0); log.add(LogType.MSG_DATA_MIME_LENGTH, 3, Long.toString(totalLength)); - String charset = bd.getCharset(); - // the charset defaults to us-ascii, but we want to default to utf-8 - if ("us-ascii".equals(charset)) { - charset = "utf-8"; - } + OpenPgpMetadata metadata; + if (charsetVerifier.isDefinitelyBinary()) { + metadata = new OpenPgpMetadata(mFilename, mimeType, 0L, totalLength); + } else { + if (charsetVerifier.isCharsetFaulty() && charsetVerifier.isCharsetGuessed()) { + log.add(LogType.MSG_DATA_MIME_CHARSET_UNKNOWN, 3, charsetVerifier.getMaybeFaultyCharset()); + } else if (charsetVerifier.isCharsetFaulty()) { + log.add(LogType.MSG_DATA_MIME_CHARSET_FAULTY, 3, charsetVerifier.getCharset()); + } else if (charsetVerifier.isCharsetGuessed()) { + log.add(LogType.MSG_DATA_MIME_CHARSET_GUESS, 3, charsetVerifier.getCharset()); + } else { + log.add(LogType.MSG_DATA_MIME_CHARSET, 3, charsetVerifier.getCharset()); + } - OpenPgpMetadata metadata = new OpenPgpMetadata(mFilename, mimeType, 0L, totalLength, charset); + metadata = new OpenPgpMetadata(mFilename, charsetVerifier.getGuessedMimeType(), 0L, totalLength, + charsetVerifier.getCharset()); + } out.close(); outputUris.add(uri); diff --git a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/results/OperationResult.java b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/results/OperationResult.java index f9c2db8e8..ec2fddbd0 100644 --- a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/results/OperationResult.java +++ b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/results/OperationResult.java @@ -860,6 +860,10 @@ public abstract class OperationResult implements Parcelable { MSG_DATA_MIME_FROM_EXTENSION (LogLevel.DEBUG, R.string.msg_data_mime_from_extension), MSG_DATA_MIME_FILENAME (LogLevel.DEBUG, R.string.msg_data_mime_filename), MSG_DATA_MIME_LENGTH (LogLevel.DEBUG, R.string.msg_data_mime_length), + MSG_DATA_MIME_CHARSET (LogLevel.DEBUG, R.string.msg_data_mime_charset), + MSG_DATA_MIME_CHARSET_FAULTY (LogLevel.WARN, R.string.msg_data_mime_charset_faulty), + MSG_DATA_MIME_CHARSET_GUESS (LogLevel.DEBUG, R.string.msg_data_mime_charset_guess), + MSG_DATA_MIME_CHARSET_UNKNOWN (LogLevel.DEBUG, R.string.msg_data_mime_charset_unknown), MSG_DATA_MIME (LogLevel.DEBUG, R.string.msg_data_mime), MSG_DATA_MIME_OK (LogLevel.INFO, R.string.msg_data_mime_ok), MSG_DATA_MIME_NONE (LogLevel.DEBUG, R.string.msg_data_mime_none), diff --git a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/pgp/PgpDecryptVerifyOperation.java b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/pgp/PgpDecryptVerifyOperation.java index c4755c7c5..e15139a7f 100644 --- a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/pgp/PgpDecryptVerifyOperation.java +++ b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/pgp/PgpDecryptVerifyOperation.java @@ -35,8 +35,6 @@ import android.support.annotation.NonNull; import android.text.TextUtils; import android.webkit.MimeTypeMap; -import org.openintents.openpgp.OpenPgpDecryptionResult; -import org.openintents.openpgp.OpenPgpMetadata; import org.bouncycastle.bcpg.ArmoredInputStream; import org.bouncycastle.openpgp.PGPCompressedData; import org.bouncycastle.openpgp.PGPDataValidationException; @@ -56,10 +54,13 @@ import org.bouncycastle.openpgp.operator.jcajce.CachingDataDecryptorFactory; import org.bouncycastle.openpgp.operator.jcajce.JcaPGPDigestCalculatorProviderBuilder; import org.bouncycastle.openpgp.operator.jcajce.JcePBEDataDecryptorFactoryBuilder; import org.bouncycastle.util.encoders.DecoderException; +import org.openintents.openpgp.OpenPgpDecryptionResult; +import org.openintents.openpgp.OpenPgpMetadata; import org.sufficientlysecure.keychain.Constants; import org.sufficientlysecure.keychain.Constants.key; import org.sufficientlysecure.keychain.R; import org.sufficientlysecure.keychain.operations.BaseOperation; +import org.sufficientlysecure.keychain.util.CharsetVerifier; import org.sufficientlysecure.keychain.operations.results.DecryptVerifyResult; import org.sufficientlysecure.keychain.operations.results.OperationResult.LogType; import org.sufficientlysecure.keychain.operations.results.OperationResult.OperationLog; @@ -387,9 +388,9 @@ public class PgpDecryptVerifyOperation extends BaseOperation 0) { // Log.d(Constants.TAG, "read bytes: " + length); if (out != null) { @@ -448,6 +449,8 @@ public class PgpDecryptVerifyOperation extends BaseOperation firstBytes.length ? firstBytes.length : length); @@ -480,18 +483,21 @@ public class PgpDecryptVerifyOperation extends BaseOperation - - + + + + diff --git a/OpenKeychain/src/main/res/values/strings.xml b/OpenKeychain/src/main/res/values/strings.xml index f32eb0269..9920d8057 100644 --- a/OpenKeychain/src/main/res/values/strings.xml +++ b/OpenKeychain/src/main/res/values/strings.xml @@ -95,7 +95,7 @@ "Add file(s)" "Share" "Open with…" - "Copy decrypted text" + "Copy to clipboard" "Read from clipboard" "Select input file" "Encrypt files" @@ -1401,6 +1401,10 @@ "Filename: '%s'" "Guessing MIME type from extension" "Content-Length: %s" + "Charset is '%s'" + "Charset is '%s', but decoding failed!" + "Charset appears to be '%s'" + "Charset is unknown, or data is not text." "Parsing MIME data structure" "Finished parsing" "No MIME structure found" diff --git a/OpenKeychain/src/test/java/org/sufficientlysecure/keychain/pgp/InputDataOperationTest.java b/OpenKeychain/src/test/java/org/sufficientlysecure/keychain/pgp/InputDataOperationTest.java index 4277649fb..fba8be061 100644 --- a/OpenKeychain/src/test/java/org/sufficientlysecure/keychain/pgp/InputDataOperationTest.java +++ b/OpenKeychain/src/test/java/org/sufficientlysecure/keychain/pgp/InputDataOperationTest.java @@ -20,6 +20,7 @@ package org.sufficientlysecure.keychain.pgp; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.FileNotFoundException; import java.io.PrintStream; import java.security.Security; import java.util.ArrayList; @@ -30,18 +31,20 @@ import android.content.ContentValues; import android.net.Uri; import junit.framework.Assert; +import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; +import org.openintents.openpgp.OpenPgpMetadata; import org.robolectric.RobolectricGradleTestRunner; import org.robolectric.RuntimeEnvironment; import org.robolectric.annotation.Config; import org.robolectric.shadows.ShadowLog; -import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.sufficientlysecure.keychain.WorkaroundBuildConfig; import org.sufficientlysecure.keychain.operations.InputDataOperation; import org.sufficientlysecure.keychain.operations.results.InputDataResult; +import org.sufficientlysecure.keychain.operations.results.OperationResult.LogType; import org.sufficientlysecure.keychain.provider.ProviderHelper; import org.sufficientlysecure.keychain.provider.TemporaryFileProvider; import org.sufficientlysecure.keychain.service.InputDataParcel; @@ -80,20 +83,20 @@ public class InputDataOperationTest { } @Test - public void testMimeDecoding () throws Exception { + public void testMimeDecoding() throws Exception { String mimeMail = "Content-Type: multipart/mixed; boundary=\"=-26BafqxfXmhVNMbYdoIi\"\n" + "\n" + "--=-26BafqxfXmhVNMbYdoIi\n" + - "Content-Type: text/plain\n" + + "Content-Type: text/plain; charset=utf-8\n" + "Content-Transfer-Encoding: quoted-printable\n" + "Content-Disposition: attachment; filename=data.txt\n" + "\n" + "message part 1\n" + "\n" + "--=-26BafqxfXmhVNMbYdoIi\n" + - "Content-Type: text/testvalue\n" + + "Content-Type: text/testvalue; charset=iso-8859-1\n" + "Content-Description: Dummy content description\n" + "\n" + "message part 2.1\n" + @@ -156,7 +159,137 @@ public class InputDataOperationTest { Assert.assertEquals("second part must have expected content", "message part 2.1\nmessage part 2.2\n", new String(outStream2.toByteArray())); + OpenPgpMetadata metadata = result.mMetadata.get(0); + Assert.assertEquals("text/plain", metadata.getMimeType()); + Assert.assertEquals("utf-8", metadata.getCharset()); + metadata = result.mMetadata.get(1); + Assert.assertEquals("text/testvalue", metadata.getMimeType()); + Assert.assertEquals("iso-8859-1", metadata.getCharset()); + } + + @Test + public void testMimeDecodingExplicitFaultyCharset() throws Exception { + + String mimeContent = "Content-Type: text/plain; charset=utf-8\n" + + "\n" + + "message with binary data in it\n"; + + byte[] data = mimeContent.getBytes(); + data[60] = (byte) 0xc3; + data[61] = (byte) 0x28; + + InputDataResult result = runSimpleDataInputOperation(data); + + // must be successful, no verification, have two output URIs + Assert.assertTrue(result.success()); + Assert.assertNull(result.mDecryptVerifyResult); + + OpenPgpMetadata metadata = result.mMetadata.get(0); + Assert.assertEquals("text/plain", metadata.getMimeType()); + + Assert.assertEquals("charset should be set since it was explicitly specified", + "utf-8", metadata.getCharset()); + Assert.assertTrue("faulty charset should have been detected", + result.getLog().containsType(LogType.MSG_DATA_MIME_CHARSET_FAULTY)); + } + + @Test + public void testMimeDecodingImplicitFaultyCharset() throws Exception { + + String mimeContent = "Content-Type: text/plain\n" + + "\n" + + "message with binary data in it\n"; + + byte[] data = mimeContent.getBytes(); + data[45] = (byte) 0xc3; + data[46] = (byte) 0x28; + + InputDataResult result = runSimpleDataInputOperation(data); + + // must be successful, no verification, have two output URIs + Assert.assertTrue(result.success()); + Assert.assertNull(result.mDecryptVerifyResult); + + OpenPgpMetadata metadata = result.mMetadata.get(0); + Assert.assertEquals("text/plain", metadata.getMimeType()); + + Assert.assertNull("charset was bad so it should not be set", metadata.getCharset()); + Assert.assertTrue("faulty charset should have been detected", + result.getLog().containsType(LogType.MSG_DATA_MIME_CHARSET_UNKNOWN)); + } + + @Test + public void testMimeDecodingImplicitGuessedCharset() throws Exception { + + String mimeContent = "Content-Type: text/plain\n" + + "\n" + + "proper, utf-8 encoded message ☭\n"; + + InputDataResult result = runSimpleDataInputOperation(mimeContent.getBytes()); + + // must be successful, no verification, have two output URIs + Assert.assertTrue(result.success()); + Assert.assertNull(result.mDecryptVerifyResult); + + OpenPgpMetadata metadata = result.mMetadata.get(0); + Assert.assertEquals("text/plain", metadata.getMimeType()); + + Assert.assertEquals("charset should be set since it was guessed and not faulty", + "utf-8", metadata.getCharset()); + Assert.assertTrue("charset should have been guessed", + result.getLog().containsType(LogType.MSG_DATA_MIME_CHARSET_GUESS)); + } + + @Test + public void testMimeDecodingOctetStreamGuessedCharset() throws Exception { + + String mimeContent = "Content-Type: application/octet-stream\n" + + "\n" + + "proper, utf-8 encoded message ☭\n"; + + InputDataResult result = runSimpleDataInputOperation(mimeContent.getBytes()); + + // must be successful, no verification, have two output URIs + Assert.assertTrue(result.success()); + Assert.assertNull(result.mDecryptVerifyResult); + + OpenPgpMetadata metadata = result.mMetadata.get(0); + Assert.assertEquals("text/plain", metadata.getMimeType()); + + Assert.assertEquals("charset should be set since it was guessed and not faulty", + "utf-8", metadata.getCharset()); + Assert.assertTrue("charset should have been guessed", + result.getLog().containsType(LogType.MSG_DATA_MIME_CHARSET_GUESS)); + } + + private InputDataResult runSimpleDataInputOperation(byte[] mimeContentBytes) throws FileNotFoundException { + ByteArrayOutputStream outStream1 = new ByteArrayOutputStream(); + ByteArrayOutputStream outStream2 = new ByteArrayOutputStream(); + ContentResolver mockResolver = mock(ContentResolver.class); + + // fake openOutputStream first and second + when(mockResolver.openOutputStream(any(Uri.class), eq("w"))) + .thenReturn(outStream1, outStream2); + + // fake openInputStream + Uri fakeInputUri = Uri.parse("content://fake/1"); + when(mockResolver.openInputStream(fakeInputUri)).thenReturn( + new ByteArrayInputStream(mimeContentBytes)); + + Uri fakeOutputUri1 = Uri.parse("content://fake/out/1"); + when(mockResolver.insert(eq(TemporaryFileProvider.CONTENT_URI), any(ContentValues.class))) + .thenReturn(fakeOutputUri1); + + // application which returns mockresolver + Application spyApplication = spy(RuntimeEnvironment.application); + when(spyApplication.getContentResolver()).thenReturn(mockResolver); + + InputDataOperation op = new InputDataOperation(spyApplication, + new ProviderHelper(RuntimeEnvironment.application), null); + + InputDataParcel input = new InputDataParcel(fakeInputUri, null); + return op.execute(input, new CryptoInputParcel()); } } diff --git a/OpenKeychain/src/test/java/org/sufficientlysecure/keychain/util/CharsetVerifierTest.java b/OpenKeychain/src/test/java/org/sufficientlysecure/keychain/util/CharsetVerifierTest.java new file mode 100644 index 000000000..a87c1595e --- /dev/null +++ b/OpenKeychain/src/test/java/org/sufficientlysecure/keychain/util/CharsetVerifierTest.java @@ -0,0 +1,144 @@ +package org.sufficientlysecure.keychain.util; + + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.robolectric.RobolectricGradleTestRunner; +import org.robolectric.annotation.Config; +import org.sufficientlysecure.keychain.WorkaroundBuildConfig; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +@RunWith(RobolectricGradleTestRunner.class) +@Config(constants = WorkaroundBuildConfig.class, sdk = 21, manifest = "src/main/AndroidManifest.xml") +public class CharsetVerifierTest { + + @Test + public void testTypeImagePngAlwaysBinary() throws Exception { + byte[] bytes = "bla bluh ☭".getBytes("utf-8"); + + CharsetVerifier charsetVerifier = new CharsetVerifier(bytes, "image/png", null); + charsetVerifier.readBytesFromBuffer(0, bytes.length); + + assertTrue("image/png should be marked as definitely binary", charsetVerifier.isDefinitelyBinary()); + assertFalse("image/png should never be marked as, even if it is", charsetVerifier.isProbablyText()); + assertNull("charset should be null", charsetVerifier.getCharset()); + } + + @Test + public void testUtf8SpecifiedButFaulty() throws Exception { + byte[] bytes = "bla bluh ☭".getBytes("utf-8"); + bytes[4] = (byte) 0xc3; + bytes[5] = (byte) 0x28; + + CharsetVerifier charsetVerifier = new CharsetVerifier(bytes, "text/something", "utf-8"); + charsetVerifier.readBytesFromBuffer(0, bytes.length); + + assertFalse("text/plain should not be marked as binary, even if it is", charsetVerifier.isDefinitelyBinary()); + assertTrue("text/plain should be marked as text, even if it isn't valid", charsetVerifier.isProbablyText()); + assertTrue("encoding contained illegal chars, so it should be marked as faulty", charsetVerifier.isCharsetFaulty()); + assertFalse("charset was specified and should not be marked as guessed", charsetVerifier.isCharsetGuessed()); + assertEquals("mimetype should be preserved", "text/something", charsetVerifier.getGuessedMimeType()); + assertEquals("charset should be utf-8 since it was given explicitly", "utf-8", charsetVerifier.getCharset()); + assertEquals("charset should be utf-8 since it was given explicitly", "utf-8", charsetVerifier.getMaybeFaultyCharset()); + } + + @Test + public void testUtf8GuessedAndFaulty() throws Exception { + byte[] bytes = "bla bluh ☭".getBytes("utf-8"); + bytes[4] = (byte) 0xc3; + bytes[5] = (byte) 0x28; + + CharsetVerifier charsetVerifier = new CharsetVerifier(bytes, "text/plain", null); + charsetVerifier.readBytesFromBuffer(0, bytes.length); + + assertFalse("text/plain should not be marked as binary, even if it is", charsetVerifier.isDefinitelyBinary()); + assertTrue("text/plain should be marked as text, even if it isn't valid", charsetVerifier.isProbablyText()); + assertTrue("encoding contained illegal chars, so it should be marked as faulty", charsetVerifier.isCharsetFaulty()); + assertTrue("charset was guessed and should be marked as such", charsetVerifier.isCharsetGuessed()); + assertNull("charset should be null since the guess was faulty", charsetVerifier.getCharset()); + assertEquals("mimetype should be set to text", "text/plain", charsetVerifier.getGuessedMimeType()); + assertEquals("maybe-faulty charset should be utf-8", "utf-8", charsetVerifier.getMaybeFaultyCharset()); + } + + @Test + public void testGuessedEncoding() throws Exception { + byte[] bytes = "bla bluh ☭".getBytes("utf-8"); + + CharsetVerifier charsetVerifier = new CharsetVerifier(bytes, "application/octet-stream", null); + charsetVerifier.readBytesFromBuffer(0, bytes.length); + + assertFalse("application/octet-stream with text content is not definitely binary", charsetVerifier.isDefinitelyBinary()); + assertTrue("application/octet-stream with text content should be probably text", charsetVerifier.isProbablyText()); + assertFalse("detected charset should not be faulty", charsetVerifier.isCharsetFaulty()); + assertTrue("charset was guessed and should be marked as such", charsetVerifier.isCharsetGuessed()); + assertEquals("mimetype should be set to text", "text/plain", charsetVerifier.getGuessedMimeType()); + assertEquals("guessed charset is utf-8", "utf-8", charsetVerifier.getCharset()); + } + + @Test + public void testWindows1252Faulty() throws Exception { + byte[] bytes = "bla bluh ☭".getBytes("windows-1252"); + bytes[2] = (byte) 0x9d; + + CharsetVerifier charsetVerifier = new CharsetVerifier(bytes, "text/plain", "windows-1252"); + charsetVerifier.readBytesFromBuffer(0, bytes.length); + + assertFalse("text/plain is never definitely binary", charsetVerifier.isDefinitelyBinary()); + assertTrue("text/plain is always probably text", charsetVerifier.isProbablyText()); + assertTrue("charset contained faulty characters", charsetVerifier.isCharsetFaulty()); + assertFalse("charset was not guessed", charsetVerifier.isCharsetGuessed()); + assertEquals("charset is returned correctly", "windows-1252", charsetVerifier.getCharset()); + } + + @Test + public void testWindows1252Good() throws Exception { + byte[] bytes = "bla bluh ☭".getBytes("windows-1252"); + // this is ‡ in windows-1252 + bytes[2] = (byte) 0x87; + + CharsetVerifier charsetVerifier = new CharsetVerifier(bytes, "text/plain", "windows-1252"); + charsetVerifier.readBytesFromBuffer(0, bytes.length); + + assertFalse("text/plain is never definitely binary", charsetVerifier.isDefinitelyBinary()); + assertTrue("text/plain is always probably text", charsetVerifier.isProbablyText()); + assertFalse("charset contained no faulty characters", charsetVerifier.isCharsetFaulty()); + assertFalse("charset was not guessed", charsetVerifier.isCharsetGuessed()); + assertEquals("charset is returned correctly", "windows-1252", charsetVerifier.getCharset()); + } + + @Test(expected = IllegalStateException.class) + public void testReadAfterGetterShouldCrash() throws Exception { + byte[] bytes = "bla bluh ☭".getBytes("utf-8"); + + CharsetVerifier charsetVerifier = new CharsetVerifier(bytes, "text/plain", null); + charsetVerifier.readBytesFromBuffer(0, bytes.length); + charsetVerifier.isCharsetFaulty(); + + charsetVerifier.readBytesFromBuffer(0, bytes.length); + } + + + @Test + public void testStaggeredInput() throws Exception { + byte[] bytes = "bla bluh ☭".getBytes("utf-8"); + bytes[4] = (byte) 0xc3; + bytes[5] = (byte) 0x28; + + CharsetVerifier charsetVerifier = new CharsetVerifier(bytes, "text/plain", null); + for (int i = 0; i < bytes.length; i++) { + charsetVerifier.readBytesFromBuffer(i, i+1); + } + + assertFalse("text/plain should not be marked as binary, even if it is", charsetVerifier.isDefinitelyBinary()); + assertTrue("text/plain should be marked as text, even if it isn't valid", charsetVerifier.isProbablyText()); + assertTrue("encoding contained illegal chars, so it should be marked as faulty", charsetVerifier.isCharsetFaulty()); + assertTrue("charset was guessed and should be marked as such", charsetVerifier.isCharsetGuessed()); + assertNull("charset should be null since the guess was faulty", charsetVerifier.getCharset()); + assertEquals("maybe-faulty charset should be utf-8", "utf-8", charsetVerifier.getMaybeFaultyCharset()); + } + +}