]> git.pld-linux.org Git - packages/obs-studio.git/blame - Remove_encrypted_RTMP_support.patch
relup
[packages/obs-studio.git] / Remove_encrypted_RTMP_support.patch
CommitLineData
2f3b3ac9
KM
1From 7d07b57993cdd8114afca6df2c4eecc619264447 Mon Sep 17 00:00:00 2001
2From: tytan652 <tytan652@tytanium.xyz>
3Date: Tue, 19 Oct 2021 15:21:32 +0200
4Subject: [PATCH] obs-outputs,librtmp: Remove encrypted RTMP support
5
6RC4 and Diffie-Hellmann Key related codes are removed
7---
8 plugins/obs-outputs/CMakeLists.txt | 2 -
9 plugins/obs-outputs/librtmp/dh.h | 384 ------------------------
10 plugins/obs-outputs/librtmp/dhgroups.h | 199 ------------
11 plugins/obs-outputs/librtmp/handshake.h | 285 +-----------------
12 plugins/obs-outputs/librtmp/rtmp.c | 45 ---
13 plugins/obs-outputs/librtmp/rtmp.h | 6 -
14 6 files changed, 1 insertion(+), 920 deletions(-)
15 delete mode 100644 plugins/obs-outputs/librtmp/dh.h
16 delete mode 100644 plugins/obs-outputs/librtmp/dhgroups.h
17
18diff --git a/plugins/obs-outputs/librtmp/dh.h b/plugins/obs-outputs/librtmp/dh.h
19deleted file mode 100644
20index 466b64e4adc0..000000000000
21--- a/plugins/obs-outputs/librtmp/dh.h
22+++ /dev/null
23@@ -1,384 +0,0 @@
24-/* RTMPDump - Diffie-Hellmann Key Exchange
25- * Copyright (C) 2009 Andrej Stepanchuk
26- * Copyright (C) 2009-2010 Howard Chu
27- *
28- * This file is part of librtmp.
29- *
30- * librtmp is free software; you can redistribute it and/or modify
31- * it under the terms of the GNU Lesser General Public License as
32- * published by the Free Software Foundation; either version 2.1,
33- * or (at your option) any later version.
34- *
35- * librtmp is distributed in the hope that it will be useful,
36- * but WITHOUT ANY WARRANTY; without even the implied warranty of
37- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
38- * GNU General Public License for more details.
39- *
40- * You should have received a copy of the GNU Lesser General Public License
41- * along with librtmp see the file COPYING. If not, write to
42- * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
43- * Boston, MA 02110-1301, USA.
44- * http://www.gnu.org/copyleft/lgpl.html
45- */
46-
47-#if defined(USE_MBEDTLS)
48-#include <mbedtls/dhm.h>
49-#include <mbedtls/bignum.h>
50-typedef mbedtls_mpi* MP_t;
51-#define MP_new(m) m = malloc(sizeof(mbedtls_mpi)); mbedtls_mpi_init(m)
52-#define MP_set_w(mpi, w) mbedtls_mpi_lset(mpi, w)
53-#define MP_cmp(u, v) mbedtls_mpi_cmp_mpi(u, v)
54-#define MP_set(u, v) mbedtls_mpi_copy(u, v)
55-#define MP_sub_w(mpi, w) mbedtls_mpi_sub_int(mpi, mpi, w)
56-#define MP_cmp_1(mpi) mbedtls_mpi_cmp_int(mpi, 1)
57-#define MP_modexp(r, y, q, p) mbedtls_mpi_exp_mod(r, y, q, p, NULL)
58-#define MP_free(mpi) mbedtls_mpi_free(mpi); free(mpi)
59-#define MP_gethex(u, hex, res) MP_new(u); res = mbedtls_mpi_read_string(u, 16, hex) == 0
60-#define MP_bytes(u) mbedtls_mpi_size(u)
61-#define MP_setbin(u,buf,len) mbedtls_mpi_write_binary(u,buf,len)
62-#define MP_getbin(u,buf,len) MP_new(u); mbedtls_mpi_read_binary(u,buf,len)
63-
64-typedef struct MDH
65-{
66- MP_t p;
67- MP_t g;
68- MP_t pub_key;
69- MP_t priv_key;
70- long length;
71- mbedtls_dhm_context ctx;
72-} MDH;
73-
74-#define MDH_new() calloc(1,sizeof(MDH))
75-#define MDH_free(vp) {MDH *_dh = vp; mbedtls_dhm_free(&_dh->ctx); MP_free(_dh->p); MP_free(_dh->g); MP_free(_dh->pub_key); MP_free(_dh->priv_key); free(_dh);}
76-
77-static int MDH_generate_key(RTMP *r, MDH *dh)
78-{
79- unsigned char out[2];
80- MP_set(&dh->ctx.P, dh->p);
81- MP_set(&dh->ctx.G, dh->g);
82- dh->ctx.len = 128;
83- mbedtls_dhm_make_public(&dh->ctx, 1024, out, 1, mbedtls_ctr_drbg_random, &r->RTMP_TLS_ctx->ctr_drbg);
84- MP_new(dh->pub_key);
85- MP_new(dh->priv_key);
86- MP_set(dh->pub_key, &dh->ctx.GX);
87- MP_set(dh->priv_key, &dh->ctx.X);
88- return 1;
89-}
90-
91-static int MDH_compute_key(uint8_t *secret, size_t len, MP_t pub, MDH *dh)
92-{
93- MP_set(&dh->ctx.GY, pub);
94- size_t olen;
95- mbedtls_dhm_calc_secret(&dh->ctx, secret, len, &olen, NULL, NULL);
96- return 0;
97-}
98-
99-#elif defined(USE_POLARSSL)
100-#include <polarssl/dhm.h>
101-typedef mpi * MP_t;
102-#define MP_new(m) m = malloc(sizeof(mpi)); mpi_init(m)
103-#define MP_set_w(mpi, w) mpi_lset(mpi, w)
104-#define MP_cmp(u, v) mpi_cmp_mpi(u, v)
105-#define MP_set(u, v) mpi_copy(u, v)
106-#define MP_sub_w(mpi, w) mpi_sub_int(mpi, mpi, w)
107-#define MP_cmp_1(mpi) mpi_cmp_int(mpi, 1)
108-#define MP_modexp(r, y, q, p) mpi_exp_mod(r, y, q, p, NULL)
109-#define MP_free(mpi) mpi_free(mpi); free(mpi)
110-#define MP_gethex(u, hex, res) MP_new(u); res = mpi_read_string(u, 16, hex) == 0
111-#define MP_bytes(u) mpi_size(u)
112-#define MP_setbin(u,buf,len) mpi_write_binary(u,buf,len)
113-#define MP_getbin(u,buf,len) MP_new(u); mpi_read_binary(u,buf,len)
114-
115-typedef struct MDH
116-{
117- MP_t p;
118- MP_t g;
119- MP_t pub_key;
120- MP_t priv_key;
121- long length;
122- dhm_context ctx;
123-} MDH;
124-
125-#define MDH_new() calloc(1,sizeof(MDH))
126-#define MDH_free(vp) {MDH *_dh = vp; dhm_free(&_dh->ctx); MP_free(_dh->p); MP_free(_dh->g); MP_free(_dh->pub_key); MP_free(_dh->priv_key); free(_dh);}
127-
128-static int MDH_generate_key(MDH *dh)
129-{
130- unsigned char out[2];
131- MP_set(&dh->ctx.P, dh->p);
132- MP_set(&dh->ctx.G, dh->g);
133- dh->ctx.len = 128;
134- dhm_make_public(&dh->ctx, 1024, out, 1, havege_random, &RTMP_TLS_ctx->hs);
135- MP_new(dh->pub_key);
136- MP_new(dh->priv_key);
137- MP_set(dh->pub_key, &dh->ctx.GX);
138- MP_set(dh->priv_key, &dh->ctx.X);
139- return 1;
140-}
141-
142-static int MDH_compute_key(uint8_t *secret, size_t len, MP_t pub, MDH *dh)
143-{
144- MP_set(&dh->ctx.GY, pub);
145- dhm_calc_secret(&dh->ctx, secret, &len);
146- return 0;
147-}
148-
149-#elif defined(USE_GNUTLS)
150-#include <gmp.h>
151-#include <nettle/bignum.h>
152-typedef mpz_ptr MP_t;
153-#define MP_new(m) m = malloc(sizeof(*m)); mpz_init2(m, 1)
154-#define MP_set_w(mpi, w) mpz_set_ui(mpi, w)
155-#define MP_cmp(u, v) mpz_cmp(u, v)
156-#define MP_set(u, v) mpz_set(u, v)
157-#define MP_sub_w(mpi, w) mpz_sub_ui(mpi, mpi, w)
158-#define MP_cmp_1(mpi) mpz_cmp_ui(mpi, 1)
159-#define MP_modexp(r, y, q, p) mpz_powm(r, y, q, p)
160-#define MP_free(mpi) mpz_clear(mpi); free(mpi)
161-#define MP_gethex(u, hex, res) u = malloc(sizeof(*u)); mpz_init2(u, 1); res = (mpz_set_str(u, hex, 16) == 0)
162-#define MP_bytes(u) (mpz_sizeinbase(u, 2) + 7) / 8
163-#define MP_setbin(u,buf,len) nettle_mpz_get_str_256(len,buf,u)
164-#define MP_getbin(u,buf,len) u = malloc(sizeof(*u)); mpz_init2(u, 1); nettle_mpz_set_str_256_u(u,len,buf)
165-
166-typedef struct MDH
167-{
168- MP_t p;
169- MP_t g;
170- MP_t pub_key;
171- MP_t priv_key;
172- long length;
173-} MDH;
174-
175-#define MDH_new() calloc(1,sizeof(MDH))
176-#define MDH_free(dh) do {MP_free(((MDH*)(dh))->p); MP_free(((MDH*)(dh))->g); MP_free(((MDH*)(dh))->pub_key); MP_free(((MDH*)(dh))->priv_key); free(dh);} while(0)
177-
178-extern MP_t gnutls_calc_dh_secret(MP_t *priv, MP_t g, MP_t p);
179-extern MP_t gnutls_calc_dh_key(MP_t y, MP_t x, MP_t p);
180-
181-#define MDH_generate_key(dh) (dh->pub_key = gnutls_calc_dh_secret(&dh->priv_key, dh->g, dh->p))
182-static int MDH_compute_key(uint8_t *secret, size_t len, MP_t pub, MDH *dh)
183-{
184- MP_t sec = gnutls_calc_dh_key(pub, dh->priv_key, dh->p);
185- if (sec)
186- {
187- MP_setbin(sec, secret, len);
188- MP_free(sec);
189- return 0;
190- }
191- else
192- return -1;
193-}
194-
195-#else /* USE_OPENSSL */
196-#include <openssl/bn.h>
197-#include <openssl/dh.h>
198-
199-typedef BIGNUM * MP_t;
200-#define MP_new(m) m = BN_new()
201-#define MP_set_w(mpi, w) BN_set_word(mpi, w)
202-#define MP_cmp(u, v) BN_cmp(u, v)
203-#define MP_set(u, v) BN_copy(u, v)
204-#define MP_sub_w(mpi, w) BN_sub_word(mpi, w)
205-#define MP_cmp_1(mpi) BN_cmp(mpi, BN_value_one())
206-#define MP_modexp(r, y, q, p) do {BN_CTX *ctx = BN_CTX_new(); BN_mod_exp(r, y, q, p, ctx); BN_CTX_free(ctx);} while(0)
207-#define MP_free(mpi) BN_free(mpi)
208-#define MP_gethex(u, hex, res) res = BN_hex2bn(&u, hex)
209-#define MP_bytes(u) BN_num_bytes(u)
210-#define MP_setbin(u,buf,len) BN_bn2bin(u,buf)
211-#define MP_getbin(u,buf,len) u = BN_bin2bn(buf,len,0)
212-
213-#define MDH DH
214-#define MDH_new() DH_new()
215-#define MDH_free(dh) DH_free(dh)
216-#define MDH_generate_key(dh) DH_generate_key(dh)
217-#define MDH_compute_key(secret, seclen, pub, dh) DH_compute_key(secret, pub, dh)
218-
219-#endif
220-
221-#include "log.h"
222-#include "dhgroups.h"
223-
224-/* RFC 2631, Section 2.1.5, http://www.ietf.org/rfc/rfc2631.txt */
225-static int
226-isValidPublicKey(MP_t y, MP_t p, MP_t q)
227-{
228- int ret = TRUE;
229- MP_t bn;
230- assert(y);
231-
232- MP_new(bn);
233- assert(bn);
234-
235- /* y must lie in [2,p-1] */
236- MP_set_w(bn, 1);
237- if (MP_cmp(y, bn) < 0)
238- {
239- RTMP_Log(RTMP_LOGERROR, "DH public key must be at least 2");
240- ret = FALSE;
241- goto failed;
242- }
243-
244- /* bn = p-2 */
245- MP_set(bn, p);
246- MP_sub_w(bn, 1);
247- if (MP_cmp(y, bn) > 0)
248- {
249- RTMP_Log(RTMP_LOGERROR, "DH public key must be at most p-2");
250- ret = FALSE;
251- goto failed;
252- }
253-
254- /* Verify with Sophie-Germain prime
255- *
256- * This is a nice test to make sure the public key position is calculated
257- * correctly. This test will fail in about 50% of the cases if applied to
258- * random data.
259- */
260- if (q)
261- {
262- /* y must fulfill y^q mod p = 1 */
263- MP_modexp(bn, y, q, p);
264-
265- if (MP_cmp_1(bn) != 0)
266- {
267- RTMP_Log(RTMP_LOGWARNING, "DH public key does not fulfill y^q mod p = 1");
268- }
269- }
270-
271-failed:
272- MP_free(bn);
273- return ret;
274-}
275-
276-static MDH *
277-DHInit(int nKeyBits)
278-{
279- size_t res;
280- MDH *dh = MDH_new();
281-
282- if (!dh)
283- goto failed;
284-
285- MP_new(dh->g);
286-
287- if (!dh->g)
288- goto failed;
289-
290- MP_gethex(dh->p, P1024, res); /* prime P1024, see dhgroups.h */
291- if (!res)
292- {
293- goto failed;
294- }
295-
296- MP_set_w(dh->g, 2); /* base 2 */
297-
298- dh->length = nKeyBits;
299- return dh;
300-
301-failed:
302- if (dh)
303- MDH_free(dh);
304-
305- return 0;
306-}
307-
308-static int
309-DHGenerateKey(RTMP *r)
310-{
311- MDH *dh = r->Link.dh;
312- size_t res = 0;
313- if (!dh)
314- return 0;
315-
316- while (!res)
317- {
318- MP_t q1 = NULL;
319-
320- if (!MDH_generate_key(r, dh))
321- return 0;
322-
323- MP_gethex(q1, Q1024, res);
324- assert(res);
325-
326- res = isValidPublicKey(dh->pub_key, dh->p, q1);
327- if (!res)
328- {
329- MP_free(dh->pub_key);
330- MP_free(dh->priv_key);
331- dh->pub_key = dh->priv_key = 0;
332- }
333-
334- MP_free(q1);
335- }
336- return 1;
337-}
338-
339-/* fill pubkey with the public key in BIG ENDIAN order
340- * 00 00 00 00 00 x1 x2 x3 .....
341- */
342-
343-static int
344-DHGetPublicKey(MDH *dh, uint8_t *pubkey, size_t nPubkeyLen)
345-{
346- int len;
347- if (!dh || !dh->pub_key)
348- return 0;
349-
350- len = (int)MP_bytes(dh->pub_key);
351- if (len <= 0 || len > (int) nPubkeyLen)
352- return 0;
353-
354- memset(pubkey, 0, nPubkeyLen);
355- MP_setbin(dh->pub_key, pubkey + (nPubkeyLen - len), len);
356- return 1;
357-}
358-
359-#if 0 /* unused */
360-static int
361-DHGetPrivateKey(MDH *dh, uint8_t *privkey, size_t nPrivkeyLen)
362-{
363- if (!dh || !dh->priv_key)
364- return 0;
365-
366- int len = MP_bytes(dh->priv_key);
367- if (len <= 0 || len > (int) nPrivkeyLen)
368- return 0;
369-
370- memset(privkey, 0, nPrivkeyLen);
371- MP_setbin(dh->priv_key, privkey + (nPrivkeyLen - len), len);
372- return 1;
373-}
374-#endif
375-
376-/* computes the shared secret key from the private MDH value and the
377- * other party's public key (pubkey)
378- */
379-static int
380-DHComputeSharedSecretKey(MDH *dh, uint8_t *pubkey, size_t nPubkeyLen,
381- uint8_t *secret)
382-{
383- MP_t q1 = NULL, pubkeyBn = NULL;
384- size_t len;
385- int res;
386-
387- if (!dh || !secret || nPubkeyLen >= INT_MAX)
388- return -1;
389-
390- MP_getbin(pubkeyBn, pubkey, nPubkeyLen);
391- if (!pubkeyBn)
392- return -1;
393-
394- MP_gethex(q1, Q1024, len);
395- assert(len);
396- UNUSED_PARAMETER(len); // Make GCC happy len is used in release.
397-
398- if (isValidPublicKey(pubkeyBn, dh->p, q1))
399- res = MDH_compute_key(secret, nPubkeyLen, pubkeyBn, dh);
400- else
401- res = -1;
402-
403- MP_free(q1);
404- MP_free(pubkeyBn);
405-
406- return res;
407-}
408diff --git a/plugins/obs-outputs/librtmp/dhgroups.h b/plugins/obs-outputs/librtmp/dhgroups.h
409deleted file mode 100644
410index 2db3989ce1f6..000000000000
411--- a/plugins/obs-outputs/librtmp/dhgroups.h
412+++ /dev/null
413@@ -1,199 +0,0 @@
414-/* librtmp - Diffie-Hellmann Key Exchange
415- * Copyright (C) 2009 Andrej Stepanchuk
416- *
417- * This file is part of librtmp.
418- *
419- * librtmp is free software; you can redistribute it and/or modify
420- * it under the terms of the GNU Lesser General Public License as
421- * published by the Free Software Foundation; either version 2.1,
422- * or (at your option) any later version.
423- *
424- * librtmp is distributed in the hope that it will be useful,
425- * but WITHOUT ANY WARRANTY; without even the implied warranty of
426- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
427- * GNU General Public License for more details.
428- *
429- * You should have received a copy of the GNU Lesser General Public License
430- * along with librtmp see the file COPYING. If not, write to
431- * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
432- * Boston, MA 02110-1301, USA.
433- * http://www.gnu.org/copyleft/lgpl.html
434- */
435-
436-/* from RFC 3526, see http://www.ietf.org/rfc/rfc3526.txt */
437-
438-/* 2^768 - 2 ^704 - 1 + 2^64 * { [2^638 pi] + 149686 } */
439-#define P768 \
440- "FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD1" \
441- "29024E088A67CC74020BBEA63B139B22514A08798E3404DD" \
442- "EF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245" \
443- "E485B576625E7EC6F44C42E9A63A3620FFFFFFFFFFFFFFFF"
444-
445-/* 2^1024 - 2^960 - 1 + 2^64 * { [2^894 pi] + 129093 } */
446-#define P1024 \
447- "FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD1" \
448- "29024E088A67CC74020BBEA63B139B22514A08798E3404DD" \
449- "EF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245" \
450- "E485B576625E7EC6F44C42E9A637ED6B0BFF5CB6F406B7ED" \
451- "EE386BFB5A899FA5AE9F24117C4B1FE649286651ECE65381" \
452- "FFFFFFFFFFFFFFFF"
453-
454-/* Group morder largest prime factor: */
455-#define Q1024 \
456- "7FFFFFFFFFFFFFFFE487ED5110B4611A62633145C06E0E68" \
457- "948127044533E63A0105DF531D89CD9128A5043CC71A026E" \
458- "F7CA8CD9E69D218D98158536F92F8A1BA7F09AB6B6A8E122" \
459- "F242DABB312F3F637A262174D31BF6B585FFAE5B7A035BF6" \
460- "F71C35FDAD44CFD2D74F9208BE258FF324943328F67329C0" \
461- "FFFFFFFFFFFFFFFF"
462-
463-/* 2^1536 - 2^1472 - 1 + 2^64 * { [2^1406 pi] + 741804 } */
464-#define P1536 \
465- "FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD1" \
466- "29024E088A67CC74020BBEA63B139B22514A08798E3404DD" \
467- "EF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245" \
468- "E485B576625E7EC6F44C42E9A637ED6B0BFF5CB6F406B7ED" \
469- "EE386BFB5A899FA5AE9F24117C4B1FE649286651ECE45B3D" \
470- "C2007CB8A163BF0598DA48361C55D39A69163FA8FD24CF5F" \
471- "83655D23DCA3AD961C62F356208552BB9ED529077096966D" \
472- "670C354E4ABC9804F1746C08CA237327FFFFFFFFFFFFFFFF"
473-
474-/* 2^2048 - 2^1984 - 1 + 2^64 * { [2^1918 pi] + 124476 } */
475-#define P2048 \
476- "FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD1" \
477- "29024E088A67CC74020BBEA63B139B22514A08798E3404DD" \
478- "EF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245" \
479- "E485B576625E7EC6F44C42E9A637ED6B0BFF5CB6F406B7ED" \
480- "EE386BFB5A899FA5AE9F24117C4B1FE649286651ECE45B3D" \
481- "C2007CB8A163BF0598DA48361C55D39A69163FA8FD24CF5F" \
482- "83655D23DCA3AD961C62F356208552BB9ED529077096966D" \
483- "670C354E4ABC9804F1746C08CA18217C32905E462E36CE3B" \
484- "E39E772C180E86039B2783A2EC07A28FB5C55DF06F4C52C9" \
485- "DE2BCBF6955817183995497CEA956AE515D2261898FA0510" \
486- "15728E5A8AACAA68FFFFFFFFFFFFFFFF"
487-
488-/* 2^3072 - 2^3008 - 1 + 2^64 * { [2^2942 pi] + 1690314 } */
489-#define P3072 \
490- "FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD1" \
491- "29024E088A67CC74020BBEA63B139B22514A08798E3404DD" \
492- "EF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245" \
493- "E485B576625E7EC6F44C42E9A637ED6B0BFF5CB6F406B7ED" \
494- "EE386BFB5A899FA5AE9F24117C4B1FE649286651ECE45B3D" \
495- "C2007CB8A163BF0598DA48361C55D39A69163FA8FD24CF5F" \
496- "83655D23DCA3AD961C62F356208552BB9ED529077096966D" \
497- "670C354E4ABC9804F1746C08CA18217C32905E462E36CE3B" \
498- "E39E772C180E86039B2783A2EC07A28FB5C55DF06F4C52C9" \
499- "DE2BCBF6955817183995497CEA956AE515D2261898FA0510" \
500- "15728E5A8AAAC42DAD33170D04507A33A85521ABDF1CBA64" \
501- "ECFB850458DBEF0A8AEA71575D060C7DB3970F85A6E1E4C7" \
502- "ABF5AE8CDB0933D71E8C94E04A25619DCEE3D2261AD2EE6B" \
503- "F12FFA06D98A0864D87602733EC86A64521F2B18177B200C" \
504- "BBE117577A615D6C770988C0BAD946E208E24FA074E5AB31" \
505- "43DB5BFCE0FD108E4B82D120A93AD2CAFFFFFFFFFFFFFFFF"
506-
507-/* 2^4096 - 2^4032 - 1 + 2^64 * { [2^3966 pi] + 240904 } */
508-#define P4096 \
509- "FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD1" \
510- "29024E088A67CC74020BBEA63B139B22514A08798E3404DD" \
511- "EF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245" \
512- "E485B576625E7EC6F44C42E9A637ED6B0BFF5CB6F406B7ED" \
513- "EE386BFB5A899FA5AE9F24117C4B1FE649286651ECE45B3D" \
514- "C2007CB8A163BF0598DA48361C55D39A69163FA8FD24CF5F" \
515- "83655D23DCA3AD961C62F356208552BB9ED529077096966D" \
516- "670C354E4ABC9804F1746C08CA18217C32905E462E36CE3B" \
517- "E39E772C180E86039B2783A2EC07A28FB5C55DF06F4C52C9" \
518- "DE2BCBF6955817183995497CEA956AE515D2261898FA0510" \
519- "15728E5A8AAAC42DAD33170D04507A33A85521ABDF1CBA64" \
520- "ECFB850458DBEF0A8AEA71575D060C7DB3970F85A6E1E4C7" \
521- "ABF5AE8CDB0933D71E8C94E04A25619DCEE3D2261AD2EE6B" \
522- "F12FFA06D98A0864D87602733EC86A64521F2B18177B200C" \
523- "BBE117577A615D6C770988C0BAD946E208E24FA074E5AB31" \
524- "43DB5BFCE0FD108E4B82D120A92108011A723C12A787E6D7" \
525- "88719A10BDBA5B2699C327186AF4E23C1A946834B6150BDA" \
526- "2583E9CA2AD44CE8DBBBC2DB04DE8EF92E8EFC141FBECAA6" \
527- "287C59474E6BC05D99B2964FA090C3A2233BA186515BE7ED" \
528- "1F612970CEE2D7AFB81BDD762170481CD0069127D5B05AA9" \
529- "93B4EA988D8FDDC186FFB7DC90A6C08F4DF435C934063199" \
530- "FFFFFFFFFFFFFFFF"
531-
532-/* 2^6144 - 2^6080 - 1 + 2^64 * { [2^6014 pi] + 929484 } */
533-#define P6144 \
534- "FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD1" \
535- "29024E088A67CC74020BBEA63B139B22514A08798E3404DD" \
536- "EF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245" \
537- "E485B576625E7EC6F44C42E9A637ED6B0BFF5CB6F406B7ED" \
538- "EE386BFB5A899FA5AE9F24117C4B1FE649286651ECE45B3D" \
539- "C2007CB8A163BF0598DA48361C55D39A69163FA8FD24CF5F" \
540- "83655D23DCA3AD961C62F356208552BB9ED529077096966D" \
541- "670C354E4ABC9804F1746C08CA18217C32905E462E36CE3B" \
542- "E39E772C180E86039B2783A2EC07A28FB5C55DF06F4C52C9" \
543- "DE2BCBF6955817183995497CEA956AE515D2261898FA0510" \
544- "15728E5A8AAAC42DAD33170D04507A33A85521ABDF1CBA64" \
545- "ECFB850458DBEF0A8AEA71575D060C7DB3970F85A6E1E4C7" \
546- "ABF5AE8CDB0933D71E8C94E04A25619DCEE3D2261AD2EE6B" \
547- "F12FFA06D98A0864D87602733EC86A64521F2B18177B200C" \
548- "BBE117577A615D6C770988C0BAD946E208E24FA074E5AB31" \
549- "43DB5BFCE0FD108E4B82D120A92108011A723C12A787E6D7" \
550- "88719A10BDBA5B2699C327186AF4E23C1A946834B6150BDA" \
551- "2583E9CA2AD44CE8DBBBC2DB04DE8EF92E8EFC141FBECAA6" \
552- "287C59474E6BC05D99B2964FA090C3A2233BA186515BE7ED" \
553- "1F612970CEE2D7AFB81BDD762170481CD0069127D5B05AA9" \
554- "93B4EA988D8FDDC186FFB7DC90A6C08F4DF435C934028492" \
555- "36C3FAB4D27C7026C1D4DCB2602646DEC9751E763DBA37BD" \
556- "F8FF9406AD9E530EE5DB382F413001AEB06A53ED9027D831" \
557- "179727B0865A8918DA3EDBEBCF9B14ED44CE6CBACED4BB1B" \
558- "DB7F1447E6CC254B332051512BD7AF426FB8F401378CD2BF" \
559- "5983CA01C64B92ECF032EA15D1721D03F482D7CE6E74FEF6" \
560- "D55E702F46980C82B5A84031900B1C9E59E7C97FBEC7E8F3" \
561- "23A97A7E36CC88BE0F1D45B7FF585AC54BD407B22B4154AA" \
562- "CC8F6D7EBF48E1D814CC5ED20F8037E0A79715EEF29BE328" \
563- "06A1D58BB7C5DA76F550AA3D8A1FBFF0EB19CCB1A313D55C" \
564- "DA56C9EC2EF29632387FE8D76E3C0468043E8F663F4860EE" \
565- "12BF2D5B0B7474D6E694F91E6DCC4024FFFFFFFFFFFFFFFF"
566-
567-/* 2^8192 - 2^8128 - 1 + 2^64 * { [2^8062 pi] + 4743158 } */
568-#define P8192 \
569- "FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD1" \
570- "29024E088A67CC74020BBEA63B139B22514A08798E3404DD" \
571- "EF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245" \
572- "E485B576625E7EC6F44C42E9A637ED6B0BFF5CB6F406B7ED" \
573- "EE386BFB5A899FA5AE9F24117C4B1FE649286651ECE45B3D" \
574- "C2007CB8A163BF0598DA48361C55D39A69163FA8FD24CF5F" \
575- "83655D23DCA3AD961C62F356208552BB9ED529077096966D" \
576- "670C354E4ABC9804F1746C08CA18217C32905E462E36CE3B" \
577- "E39E772C180E86039B2783A2EC07A28FB5C55DF06F4C52C9" \
578- "DE2BCBF6955817183995497CEA956AE515D2261898FA0510" \
579- "15728E5A8AAAC42DAD33170D04507A33A85521ABDF1CBA64" \
580- "ECFB850458DBEF0A8AEA71575D060C7DB3970F85A6E1E4C7" \
581- "ABF5AE8CDB0933D71E8C94E04A25619DCEE3D2261AD2EE6B" \
582- "F12FFA06D98A0864D87602733EC86A64521F2B18177B200C" \
583- "BBE117577A615D6C770988C0BAD946E208E24FA074E5AB31" \
584- "43DB5BFCE0FD108E4B82D120A92108011A723C12A787E6D7" \
585- "88719A10BDBA5B2699C327186AF4E23C1A946834B6150BDA" \
586- "2583E9CA2AD44CE8DBBBC2DB04DE8EF92E8EFC141FBECAA6" \
587- "287C59474E6BC05D99B2964FA090C3A2233BA186515BE7ED" \
588- "1F612970CEE2D7AFB81BDD762170481CD0069127D5B05AA9" \
589- "93B4EA988D8FDDC186FFB7DC90A6C08F4DF435C934028492" \
590- "36C3FAB4D27C7026C1D4DCB2602646DEC9751E763DBA37BD" \
591- "F8FF9406AD9E530EE5DB382F413001AEB06A53ED9027D831" \
592- "179727B0865A8918DA3EDBEBCF9B14ED44CE6CBACED4BB1B" \
593- "DB7F1447E6CC254B332051512BD7AF426FB8F401378CD2BF" \
594- "5983CA01C64B92ECF032EA15D1721D03F482D7CE6E74FEF6" \
595- "D55E702F46980C82B5A84031900B1C9E59E7C97FBEC7E8F3" \
596- "23A97A7E36CC88BE0F1D45B7FF585AC54BD407B22B4154AA" \
597- "CC8F6D7EBF48E1D814CC5ED20F8037E0A79715EEF29BE328" \
598- "06A1D58BB7C5DA76F550AA3D8A1FBFF0EB19CCB1A313D55C" \
599- "DA56C9EC2EF29632387FE8D76E3C0468043E8F663F4860EE" \
600- "12BF2D5B0B7474D6E694F91E6DBE115974A3926F12FEE5E4" \
601- "38777CB6A932DF8CD8BEC4D073B931BA3BC832B68D9DD300" \
602- "741FA7BF8AFC47ED2576F6936BA424663AAB639C5AE4F568" \
603- "3423B4742BF1C978238F16CBE39D652DE3FDB8BEFC848AD9" \
604- "22222E04A4037C0713EB57A81A23F0C73473FC646CEA306B" \
605- "4BCBC8862F8385DDFA9D4B7FA2C087E879683303ED5BDD3A" \
606- "062B3CF5B3A278A66D2A13F83F44F82DDF310EE074AB6A36" \
607- "4597E899A0255DC164F31CC50846851DF9AB48195DED7EA1" \
608- "B1D510BD7EE74D73FAF36BC31ECFA268359046F4EB879F92" \
609- "4009438B481C6CD7889A002ED5EE382BC9190DA6FC026E47" \
610- "9558E4475677E9AA9E3050E2765694DFC81F56E880B96E71" \
611- "60C980DD98EDD3DFFFFFFFFFFFFFFFFF"
612-
613diff --git a/plugins/obs-outputs/librtmp/handshake.h b/plugins/obs-outputs/librtmp/handshake.h
614index 7f7825592aad..0114bdec0f40 100644
615--- a/plugins/obs-outputs/librtmp/handshake.h
616+++ b/plugins/obs-outputs/librtmp/handshake.h
617@@ -26,9 +26,6 @@
618
619 #if defined(USE_MBEDTLS)
620 #include <mbedtls/md.h>
621-#if MBEDTLS_VERSION_MAJOR < 3
622-#include <mbedtls/arc4.h>
623-#endif
624 #ifndef SHA256_DIGEST_LENGTH
625 #define SHA256_DIGEST_LENGTH 32
626 #endif
627@@ -40,18 +37,8 @@ typedef mbedtls_md_context_t *HMAC_CTX;
628 #define HMAC_finish(ctx, dig) mbedtls_md_hmac_finish(ctx, dig)
629 #define HMAC_close(ctx) mbedtls_md_free(ctx); free(ctx); ctx = NULL
630
631-#if MBEDTLS_VERSION_MAJOR < 3
632-typedef mbedtls_arc4_context* RC4_handle;
633-#define RC4_alloc(h) *h = malloc(sizeof(mbedtls_arc4_context)); mbedtls_arc4_init(*h)
634-#define RC4_setkey(h,l,k) mbedtls_arc4_setup(h,k,l)
635-#define RC4_encrypt(h,l,d) mbedtls_arc4_crypt(h,l,(unsigned char *)d,(unsigned char *)d)
636-#define RC4_encrypt2(h,l,s,d) mbedtls_arc4_crypt(h,l,(unsigned char *)s,(unsigned char *)d)
637-#define RC4_free(h) mbedtls_arc4_free(h); free(h); h = NULL
638-#endif
639-
640 #elif defined(USE_POLARSSL)
641 #include <polarssl/sha2.h>
642-#include <polarssl/arc4.h>
643 #ifndef SHA256_DIGEST_LENGTH
644 #define SHA256_DIGEST_LENGTH 32
645 #endif
646@@ -60,13 +47,6 @@ typedef mbedtls_arc4_context* RC4_handle;
647 #define HMAC_crunch(ctx, buf, len) sha2_hmac_update(&ctx, buf, len)
648 #define HMAC_finish(ctx, dig) sha2_hmac_finish(&ctx, dig)
649
650-typedef arc4_context * RC4_handle;
651-#define RC4_alloc(h) *h = malloc(sizeof(arc4_context))
652-#define RC4_setkey(h,l,k) arc4_setup(h,k,l)
653-#define RC4_encrypt(h,l,d) arc4_crypt(h,l,(unsigned char *)d,(unsigned char *)d)
654-#define RC4_encrypt2(h,l,s,d) arc4_crypt(h,l,(unsigned char *)s,(unsigned char *)d)
655-#define RC4_free(h) free(h)
656-
657 #elif defined(USE_GNUTLS)
658 #include <nettle/hmac.h>
659 #include <nettle/arcfour.h>
660@@ -80,38 +60,19 @@ typedef arc4_context * RC4_handle;
661 #define HMAC_finish(ctx, dig) hmac_sha256_digest(&ctx, SHA256_DIGEST_LENGTH, dig)
662 #define HMAC_close(ctx)
663
664-typedef struct arcfour_ctx* RC4_handle;
665-#define RC4_alloc(h) *h = malloc(sizeof(struct arcfour_ctx))
666-#define RC4_setkey(h,l,k) arcfour_set_key(h, l, k)
667-#define RC4_encrypt(h,l,d) arcfour_crypt(h,l,(uint8_t *)d,(uint8_t *)d)
668-#define RC4_encrypt2(h,l,s,d) arcfour_crypt(h,l,(uint8_t *)d,(uint8_t *)s)
669-#define RC4_free(h) free(h)
670-
671 #else /* USE_OPENSSL */
672 #include <openssl/sha.h>
673 #include <openssl/hmac.h>
674-#include <openssl/rc4.h>
675 #if OPENSSL_VERSION_NUMBER < 0x0090800 || !defined(SHA256_DIGEST_LENGTH)
676 #error Your OpenSSL is too old, need 0.9.8 or newer with SHA256
677 #endif
678 #define HMAC_setup(ctx, key, len) HMAC_CTX_init(&ctx); HMAC_Init_ex(&ctx, key, len, EVP_sha256(), 0)
679 #define HMAC_crunch(ctx, buf, len) HMAC_Update(&ctx, buf, len)
680 #define HMAC_finish(ctx, dig, len) HMAC_Final(&ctx, dig, &len); HMAC_CTX_cleanup(&ctx)
681-
682-typedef RC4_KEY * RC4_handle;
683-#define RC4_alloc(h) *h = malloc(sizeof(RC4_KEY))
684-#define RC4_setkey(h,l,k) RC4_set_key(h,l,k)
685-#define RC4_encrypt(h,l,d) RC4(h,l,(uint8_t *)d,(uint8_t *)d)
686-#define RC4_encrypt2(h,l,s,d) RC4(h,l,(uint8_t *)s,(uint8_t *)d)
687-#define RC4_free(h) free(h)
688 #endif
689
690 #define FP10
691
692-#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
693-#include "dh.h"
694-#endif
695-
696 static const uint8_t GenuineFMSKey[] =
697 {
698 0x47, 0x65, 0x6e, 0x75, 0x69, 0x6e, 0x65, 0x20, 0x41, 0x64, 0x6f, 0x62,
699@@ -139,84 +100,8 @@ static const uint8_t GenuineFPKey[] =
700 0x31, 0xAE
701 }; /* 62 */
702
703-#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
704-static void InitRC4Encryption
705-(uint8_t * secretKey,
706- uint8_t * pubKeyIn,
707- uint8_t * pubKeyOut, RC4_handle *rc4keyIn, RC4_handle *rc4keyOut)
708-{
709- uint8_t digest[SHA256_DIGEST_LENGTH];
710-#if !(defined(USE_MBEDTLS) || defined(USE_POLARSSL) || defined(USE_GNUTLS))
711- unsigned int digestLen = 0;
712-#endif
713- HMAC_CTX ctx;
714-
715- RC4_alloc(rc4keyIn);
716- RC4_alloc(rc4keyOut);
717-
718- HMAC_setup(ctx, secretKey, 128);
719- HMAC_crunch(ctx, pubKeyIn, 128);
720-#if defined(USE_MBEDTLS) || defined(USE_POLARSSL) || defined(USE_GNUTLS)
721- HMAC_finish(ctx, digest);
722-#else
723- HMAC_finish(ctx, digest, digestLen);
724-#endif
725-
726- RTMP_Log(RTMP_LOGDEBUG, "RC4 Out Key: ");
727- RTMP_LogHex(RTMP_LOGDEBUG, digest, 16);
728-
729- RC4_setkey(*rc4keyOut, 16, digest);
730-
731- HMAC_setup(ctx, secretKey, 128);
732- HMAC_crunch(ctx, pubKeyOut, 128);
733-#if defined(USE_MBEDTLS) || defined(USE_POLARSSL) || defined(USE_GNUTLS)
734- HMAC_finish(ctx, digest);
735-#else
736- HMAC_finish(ctx, digest, digestLen);
737-#endif
738-
739- RTMP_Log(RTMP_LOGDEBUG, "RC4 In Key: ");
740- RTMP_LogHex(RTMP_LOGDEBUG, digest, 16);
741-
742- RC4_setkey(*rc4keyIn, 16, digest);
743-}
744-#endif
745-
746 typedef unsigned int (getoff)(uint8_t *buf, unsigned int len);
747
748-#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
749-static unsigned int
750-GetDHOffset2(uint8_t *handshake, unsigned int len)
751-{
752- (void) len;
753-
754- unsigned int offset = 0;
755- uint8_t *ptr = handshake + 768;
756- unsigned int res;
757-
758- assert(RTMP_SIG_SIZE <= len);
759-
760- offset += (*ptr);
761- ptr++;
762- offset += (*ptr);
763- ptr++;
764- offset += (*ptr);
765- ptr++;
766- offset += (*ptr);
767-
768- res = (offset % 632) + 8;
769-
770- if (res + 128 > 767)
771- {
772- RTMP_Log(RTMP_LOGERROR,
773- "%s: Couldn't calculate correct DH offset (got %d), exiting!",
774- __FUNCTION__, res);
775- exit(1);
776- }
777- return res;
778-}
779-#endif
780-
781 static unsigned int
782 GetDigestOffset2(uint8_t *handshake, unsigned int len)
783 {
784@@ -248,39 +133,6 @@ GetDigestOffset2(uint8_t *handshake, unsigned int len)
785 return res;
786 }
787
788-#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
789-static unsigned int
790-GetDHOffset1(uint8_t *handshake, unsigned int len)
791-{
792- (void) len;
793-
794- unsigned int offset = 0;
795- uint8_t *ptr = handshake + 1532;
796- unsigned int res;
797-
798- assert(RTMP_SIG_SIZE <= len);
799-
800- offset += (*ptr);
801- ptr++;
802- offset += (*ptr);
803- ptr++;
804- offset += (*ptr);
805- ptr++;
806- offset += (*ptr);
807-
808- res = (offset % 632) + 772;
809-
810- if (res + 128 > 1531)
811- {
812- RTMP_Log(RTMP_LOGERROR, "%s: Couldn't calculate DH offset (got %d), exiting!",
813- __FUNCTION__, res);
814- exit(1);
815- }
816-
817- return res;
818-}
819-#endif
820-
821 static unsigned int
822 GetDigestOffset1(uint8_t *handshake, unsigned int len)
823 {
824@@ -314,9 +166,6 @@ GetDigestOffset1(uint8_t *handshake, unsigned int len)
825 }
826
827 static getoff *digoff[] = {GetDigestOffset1, GetDigestOffset2};
828-#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
829-static getoff *dhoff[] = {GetDHOffset1, GetDHOffset2};
830-#endif
831
832 static void
833 HMACsha256(const uint8_t *message, size_t messageLen, const uint8_t *key,
834@@ -819,17 +668,9 @@ static int
835 HandShake(RTMP * r, int FP9HandShake)
836 {
837 int i, offalg = 0;
838-#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
839- int dhposClient = 0;
840-#endif
841 int digestPosClient = 0;
842 int encrypted = r->Link.protocol & RTMP_FEATURE_ENC;
843
844-#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
845- RC4_handle keyIn = 0;
846- RC4_handle keyOut = 0;
847-#endif
848-
849 #ifndef _DEBUG
850 int32_t *ip;
851 #endif
852@@ -838,71 +679,32 @@ HandShake(RTMP * r, int FP9HandShake)
853 uint8_t clientbuf[RTMP_SIG_SIZE + 4], *clientsig=clientbuf+4;
854 uint8_t serversig[RTMP_SIG_SIZE], client2[RTMP_SIG_SIZE], *reply;
855 uint8_t type;
856-#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
857- getoff *getdh = NULL;
858-#endif
859 getoff *getdig = NULL;
860
861-#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
862- if (encrypted || r->Link.SWFSize)
863- FP9HandShake = TRUE;
864- else
865- FP9HandShake = FALSE;
866-
867- r->Link.rc4keyIn = r->Link.rc4keyOut = 0;
868-#else
869 if (encrypted)
870 {
871- RTMP_Log(RTMP_LOGWARNING, "%s: encrypted RTMP is no longer supported with mbedtls 3 and later", __FUNCTION__);
872+ RTMP_Log(RTMP_LOGERROR, "%s: encrypted RTMP is not supported", __FUNCTION__);
873 return FALSE;
874 }
875 else if (r->Link.SWFSize)
876 FP9HandShake = TRUE;
877 else
878 FP9HandShake = FALSE;
879-#endif
880
881-#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
882- if (encrypted)
883- {
884- clientsig[-1] = 0x06; /* 0x08 is RTMPE as well */
885- offalg = 1;
886- }
887- else
888- clientsig[-1] = 0x03;
889-#else
890 clientsig[-1] = 0x03;
891-#endif
892
893 uptime = htonl(RTMP_GetTime());
894 memcpy(clientsig, &uptime, 4);
895
896 if (FP9HandShake)
897 {
898-#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
899- /* set version to at least 9.0.115.0 */
900- if (encrypted)
901- {
902- clientsig[4] = 128;
903- clientsig[6] = 3;
904- }
905- else
906- {
907- clientsig[4] = 10;
908- clientsig[6] = 45;
909- }
910-#else
911 clientsig[4] = 10;
912 clientsig[6] = 45;
913-#endif
914 clientsig[5] = 0;
915 clientsig[7] = 2;
916
917 RTMP_Log(RTMP_LOGDEBUG, "%s: Client type: %02X", __FUNCTION__, clientsig[-1]);
918 getdig = digoff[offalg];
919-#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
920- getdh = dhoff[offalg];
921-#endif
922 }
923 else
924 {
925@@ -921,36 +723,6 @@ HandShake(RTMP * r, int FP9HandShake)
926 /* set handshake digest */
927 if (FP9HandShake)
928 {
929-#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
930- if (encrypted)
931- {
932- /* generate Diffie-Hellmann parameters */
933- r->Link.dh = DHInit(1024);
934- if (!r->Link.dh)
935- {
936- RTMP_Log(RTMP_LOGERROR, "%s: Couldn't initialize Diffie-Hellmann!",
937- __FUNCTION__);
938- return FALSE;
939- }
940-
941- dhposClient = getdh(clientsig, RTMP_SIG_SIZE);
942- RTMP_Log(RTMP_LOGDEBUG, "%s: DH pubkey position: %d", __FUNCTION__, dhposClient);
943-
944- if (!DHGenerateKey(r))
945- {
946- RTMP_Log(RTMP_LOGERROR, "%s: Couldn't generate Diffie-Hellmann public key!",
947- __FUNCTION__);
948- return FALSE;
949- }
950-
951- if (!DHGetPublicKey(r->Link.dh, &clientsig[dhposClient], 128))
952- {
953- RTMP_Log(RTMP_LOGERROR, "%s: Couldn't write public key!", __FUNCTION__);
954- return FALSE;
955- }
956- }
957-#endif
958-
959 digestPosClient = getdig(clientsig, RTMP_SIG_SIZE); /* reuse this value in verification */
960 RTMP_Log(RTMP_LOGDEBUG, "%s: Client digest offset: %d", __FUNCTION__,
961 digestPosClient);
962@@ -1012,9 +784,6 @@ HandShake(RTMP * r, int FP9HandShake)
963 RTMP_Log(RTMP_LOGWARNING, "Trying different position for server digest!");
964 offalg ^= 1;
965 getdig = digoff[offalg];
966-#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
967- getdh = dhoff[offalg];
968-#endif
969 digestPosServer = getdig(serversig, RTMP_SIG_SIZE);
970
971 if (!VerifyDigest(digestPosServer, serversig, GenuineFMSKey, 36))
972@@ -1039,36 +808,6 @@ HandShake(RTMP * r, int FP9HandShake)
973 (uint8_t *)&r->Link.SWFVerificationResponse[10]);
974 }
975
976-#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
977- /* do Diffie-Hellmann Key exchange for encrypted RTMP */
978- if (encrypted)
979- {
980- /* compute secret key */
981- uint8_t secretKey[128] = { 0 };
982- int len, dhposServer;
983-
984- dhposServer = getdh(serversig, RTMP_SIG_SIZE);
985- RTMP_Log(RTMP_LOGDEBUG, "%s: Server DH public key offset: %d", __FUNCTION__,
986- dhposServer);
987- len = DHComputeSharedSecretKey(r->Link.dh, &serversig[dhposServer],
988- 128, secretKey);
989- if (len < 0)
990- {
991- RTMP_Log(RTMP_LOGDEBUG, "%s: Wrong secret key position!", __FUNCTION__);
992- return FALSE;
993- }
994-
995- RTMP_Log(RTMP_LOGDEBUG, "%s: Secret key: ", __FUNCTION__);
996- RTMP_LogHex(RTMP_LOGDEBUG, secretKey, 128);
997-
998- InitRC4Encryption(secretKey,
999- (uint8_t *) & serversig[dhposServer],
1000- (uint8_t *) & clientsig[dhposClient],
1001- &keyIn, &keyOut);
1002- }
1003-#endif
1004-
1005-
1006 reply = client2;
1007 #ifdef _DEBUG
1008 memset(reply, 0xff, RTMP_SIG_SIZE);
1009@@ -1195,28 +934,6 @@ HandShake(RTMP * r, int FP9HandShake)
1010 {
1011 RTMP_Log(RTMP_LOGDEBUG, "%s: Genuine Adobe Flash Media Server", __FUNCTION__);
1012 }
1013-
1014-#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
1015- if (encrypted)
1016- {
1017- char buff[RTMP_SIG_SIZE];
1018- /* set keys for encryption from now on */
1019- r->Link.rc4keyIn = keyIn;
1020- r->Link.rc4keyOut = keyOut;
1021-
1022-
1023- /* update the keystreams */
1024- if (r->Link.rc4keyIn)
1025- {
1026- RC4_encrypt(r->Link.rc4keyIn, RTMP_SIG_SIZE, (uint8_t *) buff);
1027- }
1028-
1029- if (r->Link.rc4keyOut)
1030- {
1031- RC4_encrypt(r->Link.rc4keyOut, RTMP_SIG_SIZE, (uint8_t *) buff);
1032- }
1033- }
1034-#endif
1035 }
1036 else
1037 {
1038diff --git a/plugins/obs-outputs/librtmp/rtmp.c b/plugins/obs-outputs/librtmp/rtmp.c
1039index fbc25c679526..ba86666af455 100644
1040--- a/plugins/obs-outputs/librtmp/rtmp.c
1041+++ b/plugins/obs-outputs/librtmp/rtmp.c
1042@@ -81,7 +81,6 @@ static const char *my_dhm_G = "4";
1043 #include <nettle/md5.h>
1044 #else /* USE_OPENSSL */
1045 #include <openssl/ssl.h>
1046-#include <openssl/rc4.h>
1047 #include <openssl/md5.h>
1048 #include <openssl/bio.h>
1049 #include <openssl/buffer.h>
1050@@ -1544,13 +1543,6 @@ ReadN(RTMP *r, char *buffer, int n)
1051 if (r->Link.protocol & RTMP_FEATURE_HTTP)
1052 r->m_resplen -= nBytes;
1053
1054-#if defined(CRYPTO) && (!defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3)
1055- if (r->Link.rc4keyIn)
1056- {
1057- RC4_encrypt(r->Link.rc4keyIn, nBytes, ptr);
1058- }
1059-#endif
1060-
1061 n -= nBytes;
1062 ptr += nBytes;
1063 }
1064@@ -1562,22 +1554,6 @@ static int
1065 WriteN(RTMP *r, const char *buffer, int n)
1066 {
1067 const char *ptr = buffer;
1068-#ifdef CRYPTO
1069- char *encrypted = 0;
1070- char buf[RTMP_BUFFER_CACHE_SIZE];
1071-
1072-#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
1073- if (r->Link.rc4keyOut)
1074- {
1075- if (n > (int)sizeof(buf))
1076- encrypted = (char *)malloc(n);
1077- else
1078- encrypted = (char *)buf;
1079- ptr = encrypted;
1080- RC4_encrypt2(r->Link.rc4keyOut, n, buffer, ptr);
1081- }
1082-#endif
1083-#endif
1084
1085 while (n > 0)
1086 {
1087@@ -1614,11 +1590,6 @@ WriteN(RTMP *r, const char *buffer, int n)
1088 ptr += nBytes;
1089 }
1090
1091-#ifdef CRYPTO
1092- if (encrypted && encrypted != buf)
1093- free(encrypted);
1094-#endif
1095-
1096 return n == 0;
1097 }
1098
1099@@ -4415,22 +4386,6 @@ RTMP_Close(RTMP *r)
1100 free(r->Link.tcUrl.av_val);
1101 r->Link.tcUrl.av_val = NULL;
1102 }
1103-#elif defined(CRYPTO) && (!defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3)
1104- if (r->Link.dh)
1105- {
1106- MDH_free(r->Link.dh);
1107- r->Link.dh = NULL;
1108- }
1109- if (r->Link.rc4keyIn)
1110- {
1111- RC4_free(r->Link.rc4keyIn);
1112- r->Link.rc4keyIn = NULL;
1113- }
1114- if (r->Link.rc4keyOut)
1115- {
1116- RC4_free(r->Link.rc4keyOut);
1117- r->Link.rc4keyOut = NULL;
1118- }
1119 #else
1120 for (int idx = 0; idx < r->Link.nStreams; idx++)
1121 {
1122diff --git a/plugins/obs-outputs/librtmp/rtmp.h b/plugins/obs-outputs/librtmp/rtmp.h
1123index 45090c3f1b9f..cc7e8049e644 100644
1124--- a/plugins/obs-outputs/librtmp/rtmp.h
1125+++ b/plugins/obs-outputs/librtmp/rtmp.h
1126@@ -342,12 +342,6 @@ extern "C"
1127
1128 #ifdef CRYPTO
1129 #define RTMP_SWF_HASHLEN 32
1130-#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
1131- void *dh; /* for encryption */
1132- void *rc4keyIn;
1133- void *rc4keyOut;
1134-#endif
1135-
1136 uint32_t SWFSize;
1137 uint8_t SWFHash[RTMP_SWF_HASHLEN];
1138 char SWFVerificationResponse[RTMP_SWF_HASHLEN+10];
1139--- obs-studio-27.2.4/plugins/obs-outputs/CMakeLists.txt~ 2022-03-27 23:29:23.000000000 +0000
1140+++ obs-studio-27.2.4/plugins/obs-outputs/CMakeLists.txt 2022-04-05 09:51:23.457925255 +0000
1141@@ -131,8 +131,6 @@
1142 librtmp/amf.h
1143 librtmp/bytes.h
1144 librtmp/cencode.h
1145- librtmp/dh.h
1146- librtmp/dhgroups.h
1147 librtmp/handshake.h
1148 librtmp/http.h
1149 librtmp/log.h
This page took 0.257286 seconds and 4 git commands to generate.