]> git.pld-linux.org Git - packages/obs-studio.git/blame - mbedtls3-compatibility.patch
projects / packages / obs-studio.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
relup
[packages/obs-studio.git] / mbedtls3-compatibility.patch
CommitLineData
2f3b3ac9
KM		 1From 179ad9e67b74bb8ea8d2c655ce12071c2dd67e81 Mon Sep 17 00:00:00 2001
2From: tytan652 <tytan652@tytanium.xyz>
3Date: Tue, 28 Sep 2021 18:26:23 +0200
4Subject: [PATCH] librtmp: Add mbedtls 3 compatibility
5
6Since Mbed TLS 3 doesn't support RC4 algorithm,
7encrypted RTMP is disabled if OBS is built with
8the version 3 or later of Mbed TLS.
9---
10 plugins/obs-outputs/librtmp/handshake.h | 54 ++++++++++++++++++++++++-
11 plugins/obs-outputs/librtmp/rtmp.c | 12 ++++--
12 plugins/obs-outputs/librtmp/rtmp.h | 2 +
13 3 files changed, 64 insertions(+), 4 deletions(-)
14
15diff --git a/plugins/obs-outputs/librtmp/handshake.h b/plugins/obs-outputs/librtmp/handshake.h
16index 1827867850a7..7f7825592aad 100644
17--- a/plugins/obs-outputs/librtmp/handshake.h
18+++ b/plugins/obs-outputs/librtmp/handshake.h
19@@ -26,7 +26,9 @@
20
21 #if defined(USE_MBEDTLS)
22 #include <mbedtls/md.h>
23+#if MBEDTLS_VERSION_MAJOR < 3
24 #include <mbedtls/arc4.h>
25+#endif
26 #ifndef SHA256_DIGEST_LENGTH
27 #define SHA256_DIGEST_LENGTH 32
28 #endif
29@@ -38,12 +40,14 @@ typedef mbedtls_md_context_t *HMAC_CTX;
30 #define HMAC_finish(ctx, dig) mbedtls_md_hmac_finish(ctx, dig)
31 #define HMAC_close(ctx) mbedtls_md_free(ctx); free(ctx); ctx = NULL
32
33+#if MBEDTLS_VERSION_MAJOR < 3
34 typedef mbedtls_arc4_context* RC4_handle;
35 #define RC4_alloc(h) *h = malloc(sizeof(mbedtls_arc4_context)); mbedtls_arc4_init(*h)
36 #define RC4_setkey(h,l,k) mbedtls_arc4_setup(h,k,l)
37 #define RC4_encrypt(h,l,d) mbedtls_arc4_crypt(h,l,(unsigned char *)d,(unsigned char *)d)
38 #define RC4_encrypt2(h,l,s,d) mbedtls_arc4_crypt(h,l,(unsigned char *)s,(unsigned char *)d)
39 #define RC4_free(h) mbedtls_arc4_free(h); free(h); h = NULL
40+#endif
41
42 #elif defined(USE_POLARSSL)
43 #include <polarssl/sha2.h>
44@@ -104,7 +108,9 @@ typedef RC4_KEY * RC4_handle;
45
46 #define FP10
47
48+#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
49 #include "dh.h"
50+#endif
51
52 static const uint8_t GenuineFMSKey[] =
53 {
54@@ -133,6 +139,7 @@ static const uint8_t GenuineFPKey[] =
55 0x31, 0xAE
56 }; /* 62 */
57
58+#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
59 static void InitRC4Encryption
60 (uint8_t * secretKey,
61 uint8_t * pubKeyIn,
62@@ -173,9 +180,11 @@ static void InitRC4Encryption
63
64 RC4_setkey(*rc4keyIn, 16, digest);
65 }
66+#endif
67
68 typedef unsigned int (getoff)(uint8_t *buf, unsigned int len);
69
70+#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
71 static unsigned int
72 GetDHOffset2(uint8_t *handshake, unsigned int len)
73 {
74@@ -206,6 +215,7 @@ GetDHOffset2(uint8_t *handshake, unsigned int len)
75 }
76 return res;
77 }
78+#endif
79
80 static unsigned int
81 GetDigestOffset2(uint8_t *handshake, unsigned int len)
82@@ -238,6 +248,7 @@ GetDigestOffset2(uint8_t *handshake, unsigned int len)
83 return res;
84 }
85
86+#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
87 static unsigned int
88 GetDHOffset1(uint8_t *handshake, unsigned int len)
89 {
90@@ -268,6 +279,7 @@ GetDHOffset1(uint8_t *handshake, unsigned int len)
91
92 return res;
93 }
94+#endif
95
96 static unsigned int
97 GetDigestOffset1(uint8_t *handshake, unsigned int len)
98@@ -302,7 +314,9 @@ GetDigestOffset1(uint8_t *handshake, unsigned int len)
99 }
100
101 static getoff *digoff[] = {GetDigestOffset1, GetDigestOffset2};
102+#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
103 static getoff *dhoff[] = {GetDHOffset1, GetDHOffset2};
104+#endif
105
106 static void
107 HMACsha256(const uint8_t *message, size_t messageLen, const uint8_t *key,
108@@ -805,12 +819,16 @@ static int
109 HandShake(RTMP * r, int FP9HandShake)
110 {
111 int i, offalg = 0;
112+#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
113 int dhposClient = 0;
114+#endif
115 int digestPosClient = 0;
116 int encrypted = r->Link.protocol & RTMP_FEATURE_ENC;
117
118+#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
119 RC4_handle keyIn = 0;
120 RC4_handle keyOut = 0;
121+#endif
122
123 #ifndef _DEBUG
124 int32_t *ip;
125@@ -820,15 +838,31 @@ HandShake(RTMP * r, int FP9HandShake)
126 uint8_t clientbuf[RTMP_SIG_SIZE + 4], *clientsig=clientbuf+4;
127 uint8_t serversig[RTMP_SIG_SIZE], client2[RTMP_SIG_SIZE], *reply;
128 uint8_t type;
129- getoff *getdh = NULL, *getdig = NULL;
130+#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
131+ getoff *getdh = NULL;
132+#endif
133+ getoff *getdig = NULL;
134
135+#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
136 if (encrypted || r->Link.SWFSize)
137 FP9HandShake = TRUE;
138 else
139 FP9HandShake = FALSE;
140
141 r->Link.rc4keyIn = r->Link.rc4keyOut = 0;
142+#else
143+ if (encrypted)
144+ {
145+ RTMP_Log(RTMP_LOGWARNING, "%s: encrypted RTMP is no longer supported with mbedtls 3 and later", __FUNCTION__);
146+ return FALSE;
147+ }
148+ else if (r->Link.SWFSize)
149+ FP9HandShake = TRUE;
150+ else
151+ FP9HandShake = FALSE;
152+#endif
153
154+#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
155 if (encrypted)
156 {
157 clientsig[-1] = 0x06; /* 0x08 is RTMPE as well */
158@@ -836,12 +870,16 @@ HandShake(RTMP * r, int FP9HandShake)
159 }
160 else
161 clientsig[-1] = 0x03;
162+#else
163+ clientsig[-1] = 0x03;
164+#endif
165
166 uptime = htonl(RTMP_GetTime());
167 memcpy(clientsig, &uptime, 4);
168
169 if (FP9HandShake)
170 {
171+#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
172 /* set version to at least 9.0.115.0 */
173 if (encrypted)
174 {
175@@ -853,12 +891,18 @@ HandShake(RTMP * r, int FP9HandShake)
176 clientsig[4] = 10;
177 clientsig[6] = 45;
178 }
179+#else
180+ clientsig[4] = 10;
181+ clientsig[6] = 45;
182+#endif
183 clientsig[5] = 0;
184 clientsig[7] = 2;
185
186 RTMP_Log(RTMP_LOGDEBUG, "%s: Client type: %02X", __FUNCTION__, clientsig[-1]);
187 getdig = digoff[offalg];
188+#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
189 getdh = dhoff[offalg];
190+#endif
191 }
192 else
193 {
194@@ -877,6 +921,7 @@ HandShake(RTMP * r, int FP9HandShake)
195 /* set handshake digest */
196 if (FP9HandShake)
197 {
198+#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
199 if (encrypted)
200 {
201 /* generate Diffie-Hellmann parameters */
202@@ -904,6 +949,7 @@ HandShake(RTMP * r, int FP9HandShake)
203 return FALSE;
204 }
205 }
206+#endif
207
208 digestPosClient = getdig(clientsig, RTMP_SIG_SIZE); /* reuse this value in verification */
209 RTMP_Log(RTMP_LOGDEBUG, "%s: Client digest offset: %d", __FUNCTION__,
210@@ -966,7 +1012,9 @@ HandShake(RTMP * r, int FP9HandShake)
211 RTMP_Log(RTMP_LOGWARNING, "Trying different position for server digest!");
212 offalg ^= 1;
213 getdig = digoff[offalg];
214+#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
215 getdh = dhoff[offalg];
216+#endif
217 digestPosServer = getdig(serversig, RTMP_SIG_SIZE);
218
219 if (!VerifyDigest(digestPosServer, serversig, GenuineFMSKey, 36))
220@@ -991,6 +1039,7 @@ HandShake(RTMP * r, int FP9HandShake)
221 (uint8_t *)&r->Link.SWFVerificationResponse[10]);
222 }
223
224+#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
225 /* do Diffie-Hellmann Key exchange for encrypted RTMP */
226 if (encrypted)
227 {
228@@ -1017,6 +1066,7 @@ HandShake(RTMP * r, int FP9HandShake)
229 (uint8_t *) & clientsig[dhposClient],
230 &keyIn, &keyOut);
231 }
232+#endif
233
234
235 reply = client2;
236@@ -1146,6 +1196,7 @@ HandShake(RTMP * r, int FP9HandShake)
237 RTMP_Log(RTMP_LOGDEBUG, "%s: Genuine Adobe Flash Media Server", __FUNCTION__);
238 }
239
240+#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
241 if (encrypted)
242 {
243 char buff[RTMP_SIG_SIZE];
244@@ -1165,6 +1216,7 @@ HandShake(RTMP * r, int FP9HandShake)
245 RC4_encrypt(r->Link.rc4keyOut, RTMP_SIG_SIZE, (uint8_t *) buff);
246 }
247 }
248+#endif
249 }
250 else
251 {
252diff --git a/plugins/obs-outputs/librtmp/rtmp.c b/plugins/obs-outputs/librtmp/rtmp.c
253index 70e42f7ee636..fbc25c679526 100644
254--- a/plugins/obs-outputs/librtmp/rtmp.c
255+++ b/plugins/obs-outputs/librtmp/rtmp.c
256@@ -954,7 +954,11 @@ RTMP_Connect1(RTMP *r, RTMPPacket *cp)
257
258 #if defined(USE_MBEDTLS)
259 mbedtls_net_context *server_fd = &r->RTMP_TLS_ctx->net;
260+#if MBEDTLS_VERSION_NUMBER == 0x03000000
261+ server_fd->MBEDTLS_PRIVATE(fd) = r->m_sb.sb_socket;
262+#else
263 server_fd->fd = r->m_sb.sb_socket;
264+#endif
265 TLS_setfd(r->m_sb.sb_ssl, server_fd);
266
267 // make sure we verify the certificate hostname
268@@ -1540,7 +1544,7 @@ ReadN(RTMP *r, char *buffer, int n)
269 if (r->Link.protocol & RTMP_FEATURE_HTTP)
270 r->m_resplen -= nBytes;
271
272-#ifdef CRYPTO
273+#if defined(CRYPTO) && (!defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3)
274 if (r->Link.rc4keyIn)
275 {
276 RC4_encrypt(r->Link.rc4keyIn, nBytes, ptr);
277@@ -1562,6 +1566,7 @@ WriteN(RTMP *r, const char *buffer, int n)
278 char *encrypted = 0;
279 char buf[RTMP_BUFFER_CACHE_SIZE];
280
281+#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
282 if (r->Link.rc4keyOut)
283 {
284 if (n > (int)sizeof(buf))
285@@ -1571,6 +1576,7 @@ WriteN(RTMP *r, const char *buffer, int n)
286 ptr = encrypted;
287 RC4_encrypt2(r->Link.rc4keyOut, n, buffer, ptr);
288 }
289+#endif
290 #endif
291
292 while (n > 0)
293@@ -2607,7 +2613,7 @@ b64enc(const unsigned char *input, int length, char *output, int maxsize)
294 #if defined(USE_MBEDTLS)
295 typedef mbedtls_md5_context MD5_CTX;
296
297-#if MBEDTLS_VERSION_NUMBER >= 0x02070000
298+#if MBEDTLS_VERSION_NUMBER >= 0x02070000 && MBEDTLS_VERSION_MAJOR < 3
299 #define MD5_Init(ctx) mbedtls_md5_init(ctx); mbedtls_md5_starts_ret(ctx)
300 #define MD5_Update(ctx,data,len) mbedtls_md5_update_ret(ctx,(unsigned char *)data,len)
301 #define MD5_Final(dig,ctx) mbedtls_md5_finish_ret(ctx,dig); mbedtls_md5_free(ctx)
302@@ -4409,7 +4415,7 @@ RTMP_Close(RTMP *r)
303 free(r->Link.tcUrl.av_val);
304 r->Link.tcUrl.av_val = NULL;
305 }
306-#elif defined(CRYPTO)
307+#elif defined(CRYPTO) && (!defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3)
308 if (r->Link.dh)
309 {
310 MDH_free(r->Link.dh);
311diff --git a/plugins/obs-outputs/librtmp/rtmp.h b/plugins/obs-outputs/librtmp/rtmp.h
312index 5020120ee3ee..45090c3f1b9f 100644
313--- a/plugins/obs-outputs/librtmp/rtmp.h
314+++ b/plugins/obs-outputs/librtmp/rtmp.h
315@@ -342,9 +342,11 @@ extern "C"
316
317 #ifdef CRYPTO
318 #define RTMP_SWF_HASHLEN 32
319+#if !defined(USE_MBEDTLS) || MBEDTLS_VERSION_MAJOR < 3
320 void *dh; /* for encryption */
321 void *rc4keyIn;
322 void *rc4keyOut;
323+#endif
324
325 uint32_t SWFSize;
326 uint8_t SWFHash[RTMP_SWF_HASHLEN];
OBS Studio - live streaming and screen recording software
This page took 0.137896 seconds and 4 git commands to generate.