[Jessie-discuss] IllegalArgumentException in RSA decrypt

Hello,

I am trying to use Jessie in conjunction with the Jetty Webserver. I am currently encountering an exception when trying to establish a https connection (see below). The exception is caused by a bounds check. The upper bound comes from the modulus value in the GnuRSAPrivateKey (from the RSAPrivateCrtKeySpec). The stored value is 0. I have traced this back to a read from EncodedKeyFactory::engineInit(). The value being read is indeed 0 for the modulus, but other values passed into the RSAPrivateCrtKeySpec constructor are normal looking BigInteger values (they’re large integers anyway…). I suspect that my keystore is not in a proper format for what Jessie is expecting, but I’m not quite sure how to debug the problem. I am simply using the default keystore provided by Jetty, but have tried making my own as well with the same results. I have turned on DEBUG_KEY_EXCHANGE and DEBUG_HANDSHAKE_LAYER debugging and included some of those logs below. Any help would be greatly appreciated.

Kevin

EXCEPTION:

java.lang.IllegalArgumentException: ciphertext representative out of range

at gnu.crypto.sig.rsa.RSA.decrypt(Ljava/security/PrivateKey;Ljava/math/BigInteger;)Ljava/math/BigInteger; [bytecode offset 16]

at org.metastatic.jessie.provider.SSLSocket.doServerHandshake()V [bytecode offset 3607]

at org.metastatic.jessie.provider.SSLSocket.startHandshake()V [bytecode offset 122]

at org.mortbay.jetty.security.SslSocketConnector.configure(Ljava/net/Socket;)V [bytecode offset 9]

at org.mortbay.jetty.bio.SocketConnector.accept(I)V [bytecode offset 10]

at org.mortbay.jetty.AbstractConnector$Acceptor.run()V [bytecode offset 156]

at org.mortbay.thread.BoundedThreadPool$PoolThread.run()V [bytecode offset 172]

DEBUG DATA:

13375 [main] INFO org.mortbay.log - Started SslSocketConnector @ 0.0.0.0:8443

startHandshake called in Thread[btpool0-7 - Acceptor0 SslSocketConnector @ 0.0.0.0:8443,4,main]

doing server handshake in Thread[btpool0-7 - Acceptor0 SslSocketConnector @ 0.0.0.0:8443,4,main]

struct {

type = client_hello;

struct {

version = TLSv1;

struct {

gmt_unix_time = 0;

random_bytes = 00:00:00:00:00:00:00:00:00:00:00:00:61:c7:e2:3a:69:4c:48:69:02:4d:e3:15:a9:24:52:bf;

} Random;

sessionId = ;

cipherSuites = {

TLS_RSA_WITH_RC4_128_MD5

TLS_RSA_WITH_RC4_128_SHA

TLS_RSA_WITH_3DES_EDE_CBC_SHA

TLS_RSA_WITH_DES_CBC_SHA

UNKNOWN { 0, 100 }

UNKNOWN { 0, 98 }

TLS_RSA_EXPORT_WITH_RC4_40_MD5

UNKNOWN { 0, 6 }

TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA

TLS_DHE_DSS_WITH_DES_CBC_SHA

UNKNOWN { 0, 99 }

};

compressionMethods = { null };

} ClientHello;

} Handshake;

selectSuite req=[TLS_RSA_WITH_RC4_128_MD5, TLS_RSA_WITH_RC4_128_SHA, TLS_RSA_WITH_3DES_EDE_CBC_SHA, TLS_RSA_WITH_DES_CBC_SHA, UNKNOWN { 0, 100 }, UNKNOWN { 0, 98 }, TLS_RSA_EXPORT_WITH_RC4_40_MD5, UNKNOWN { 0, 6 }, TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA, TLS_DHE_DSS_WITH_DES_CBC_SHA, UNKNOWN { 0, 99 }] avail=[TLS_DHE_DSS_WITH_AES_256_CBC_SHA, TLS_DH_DSS_WITH_AES_256_CBC_SHA, TLS_DH_RSA_WITH_AES_256_CBC_SHA, TLS_DHE_DSS_WITH_AES_128_CBC_SHA, TLS_DH_DSS_WITH_AES_128_CBC_SHA, TLS_DH_RSA_WITH_AES_128_CBC_SHA, TLS_RSA_WITH_AES_128_CBC_SHA, TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA, TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA, TLS_DH_DSS_WITH_3DES_EDE_CBC_SHA, TLS_DH_RSA_WITH_3DES_EDE_CBC_SHA, TLS_RSA_WITH_RC4_128_SHA, TLS_DH_DSS_WITH_DES_CBC_SHA, TLS_DH_RSA_WITH_DES_CBC_SHA, TLS_RSA_WITH_DES_CBC_SHA, TLS_DH_DSS_EXPORT_WITH_DES40_CBC_SHA, TLS_DH_RSA_EXPORT_WITH_DES40_CBC_SHA, TLS_RSA_EXPORT_WITH_DES40_CBC_SHA, TLS_RSA_EXPORT_WITH_RC4_40_MD5, TLS_DHE_DSS_EXPORT_WITH_DES40_CBC_SHA, TLS_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA, TLS_RSA_WITH_NULL_MD5, TLS_RSA_WITH_NULL_SHA]

TLS_RSA_WITH_RC4_128_SHA == TLS_RSA_WITH_RC4_128_SHA

saved sessions = []

rejected session

have session id? false

saved sessions = []

struct {

type = server_hello;

struct {

version = TLSv1;

struct {

gmt_unix_time = 1156956168;

random_bytes = 71:d9:f1:ca:0c:ac:6e:86:c2:ff:04:be:e4:50:4f:f6:71:0d:b1:5a:0e:5e:da:18:49:f0:9d:bf;

} Random;

sessionId = 25:d7:54:a7:e2:1e:ef:fc:ee:35:a4:dc:33:41:c6:f3:2d:34:13:1a:7e:db:b3:dc:25:5a:c2:de:5b:78:31:6e;

cipherSuite = TLS_RSA_WITH_RC4_128_SHA;

compressionMethod = null;

} ServerHello;

} Handshake;

struct {

type = certificate;

struct {

certificateList =

java.security.cert.X509Certificate {

TBSCertificate {

version = 1;

serialNo = 972222944;

signature = {

algorithm = MD5withRSA;

parameters = null;

}

issuer = C=Unknown,ST=Unknown,L=Unknown,O=Mort Bay Consulting,OU=Open Source Development,CN=Jetty Server;

validity = {

notBefore = Sun Oct 22 15:55:44 GMT+02:00 2000;

notAfter = Sat Jan 20 15:55:44 GMT+02:00 2001;

}

subject = C=Unknown,ST=Unknown,L=Unknown,O=Mort Bay Consulting,OU=Open Source Development,CN=Jetty Server;

subjectPublicKeyInfo = {

algorithm = RSA

key =

00000000 30 81 9e 30 0d 06 09 2a 86 48 86 f7 0d 01 01 01 0..0...*.H......

00000010 05 00 03 8d 00 30 81 89 02 81 81 00 b5 f5 34 e9 .....0........4.

00000020 8a b4 2a 08 3f d4 f0 7f 92 80 54 56 cb 68 55 95 ..*.?.....TV.hU.

00000030 ee 9e 7a 69 84 91 2d b8 2e 0c 48 3f d3 a0 1c b0 ..zi..-...H?....

00000040 10 1b 71 80 7f a4 ab 15 39 3e 8b 98 f8 3f 2a ec ..q.....9>...?*.

00000050 5d 9e 9a 19 c0 52 40 4e 04 1c 62 07 91 61 49 5c address@hidden

00000060 40 10 c1 60 c1 15 b6 c5 8e 0a a4 4c c7 7e 83 6f @..`.......L.~.o

00000070 e0 3d 3d 75 7a 87 a5 be d0 ca d5 bb c2 1d 49 09 .==uz.........I.

00000080 6c 09 fc dc 45 ca e0 cd be 12 72 b8 a1 c8 4e df l...E.....r...N.

00000090 54 2b 96 34 4a af 50 64 dd f5 60 11 02 03 01 00 T+.4J.Pd..`.....

000000a0 01 .

};

issuerUniqueId = null;

subjectUniqueId = null;

extensions = {

}

signatureAlgorithm = MD5withRSA;

signatureValue =

00000000 3d 11 36 42 6b d1 98 ea c0 40 da d0 aa 0f db 3e address@hidden>

00000010 68 b5 54 f1 38 a3 56 e4 24 47 77 63 5c 32 e8 e1 h.T.8.V.$Gwc\2..

00000020 bf 48 7a 25 a6 c9 65 b3 a3 ea 4e 17 73 22 ee c3 .Hz%..e...N.s"..

00000030 b8 d6 b6 68 96 2e 5b 95 37 a9 43 a4 f0 be 7c e0 ...h..[.7.C...|.

00000040 05 78 e4 35 bb 6d 70 93 ca 1b 37 3d aa 0f 96 4f .x.5.mp...7=...O

00000050 9d f0 ab 67 69 89 81 3f 51 b1 43 cd 44 c6 01 aa ...gi..?Q.C.D...

00000060 4f bb b4 ce 41 4c 72 9b cd 50 80 ff 89 70 3e e7 O...ALr..P...p>.

00000070 be b4 f8 56 5b cf d7 bc 17 71 22 78 3f 6a 73 3e ...V[....q"x?js>

}

} Certificate;

} Handshake;

send SERVER_HELLO_DONE

struct {

type = server_hello_done;

} Handshake;

>>> CSPRNG: adding random bytes:

>>> CSPRNG: 0e000000

>>> CSPRNG: adding random bytes:

>>> CSPRNG: 10000082008081793eebed4aef30c0302b1846655b72282f0694c4c7b2fed3b456c907f4c1320d4b6a3c56f21c6e8111617ab13426e530ba05d79d5df725cd59b487c0acd1c1101ba6ccd1be55dce4290d1ca82bdb5dda6da7aea2abd0a04ed81ead61e1c0d88a1fdf4f34a4d36b3e26540fee5dcee486965e274476528ff8c6703d040df624

RSA = struct {

type = client_key_exchange;

struct {

encryptedPreMasterSecret =

00000000 81 79 3e eb ed 4a ef 30 c0 30 2b 18 46 65 5b 72 .y>..J.0.0+.Fe[r

00000010 28 2f 06 94 c4 c7 b2 fe d3 b4 56 c9 07 f4 c1 32 (/........V....2

00000020 0d 4b 6a 3c 56 f2 1c 6e 81 11 61 7a b1 34 26 e5 .Kj<V..n..az.4&.

00000030 30 ba 05 d7 9d 5d f7 25 cd 59 b4 87 c0 ac d1 c1 0....].%.Y......

00000040 10 1b a6 cc d1 be 55 dc e4 29 0d 1c a8 2b db 5d ......U..)...+.]

00000050 da 6d a7 ae a2 ab d0 a0 4e d8 1e ad 61 e1 c0 d8 .m......N...a...

00000060 8a 1f df 4f 34 a4 d3 6b 3e 26 54 0f ee 5d ce e4 ...O4..k>&T..]..

00000070 86 96 5e 27 44 76 52 8f f8 c6 70 3d 04 0d f6 24 ..^'DvR...p=...$

} ClientKeyExchange;