check malloc/calloc and don't assert

Signed-off-by: Jean-Baptiste Kempf <jb@videolan.org>

src/hmac_md5.c

@@ -52,7 +52,8 @@ unsigned char *HMAC_MD5(const void *key, size_t key_len, const void *msg,
 
     // Concatenate inner padded key with message
     cat = malloc(msg_len + 64);
-    assert(cat != NULL);
+    if (!cat)
+        return NULL;
     memcpy(cat, i_key_pad, 64);
     memcpy(cat + 64, msg, msg_len);
 

src/netbios_ns.c

@@ -96,7 +96,9 @@ netbios_ns  *netbios_ns_new()
 {
     netbios_ns  *ns;
 
-    assert(ns = calloc(1, sizeof(netbios_ns)));
+    ns = calloc(1, sizeof(netbios_ns));
+    if (!ns)
+        return NULL;
     ns->abort_pipe[0] = ns->abort_pipe[1] -1;
 
     if (!ns_open_socket(ns) || ns_open_abort_pipe(ns) == -1)

src/netbios_ns_entry.c

@@ -67,7 +67,8 @@ netbios_ns_entry *netbios_ns_entry_add(netbios_ns *ns, const char *name,
     netbios_ns_entry  *entry;
 
     entry = calloc(1, sizeof(netbios_ns_entry));
-    assert(entry != NULL);
+    if (!entry)
+        return NULL;
 
     if (name != NULL)
     {

src/netbios_query.c

@@ -30,10 +30,14 @@ netbios_query   *netbios_query_new(size_t payload_size,
     netbios_query *q;
 
     q = calloc(1, sizeof(netbios_query));
-    assert(q);
+    if (!q)
+        return NULL;
 
     q->packet = calloc(1, sizeof(netbios_query_packet) + payload_size);
-    assert(q->packet);
+    if (!q->packet) {
+        free(q);
+        return NULL;
+    }
 
     q->payload_size = payload_size;
 

src/netbios_session.c

@@ -59,7 +59,8 @@ static int        session_buffer_realloc(netbios_session *s, size_t new_size)
         s->packet_payload_size = new_size;
         s->packet = new_ptr;
         return (1);
-    }
+    } else
+        free(s->packet);
     return (0);
 }
 
@@ -69,12 +70,16 @@ netbios_session *netbios_session_new(size_t buf_size)
     size_t            packet_size;
 
     session = (netbios_session *)calloc(1, sizeof(netbios_session));
-    assert(session != NULL);
+    if (!session)
+        return NULL;
 
     session->packet_payload_size = buf_size;
     packet_size = sizeof(netbios_session_packet) + session->packet_payload_size;
     session->packet = (netbios_session_packet *)malloc(packet_size);
-    assert(session->packet != NULL);
+    if (!session->packet) {
+        free(session);
+        return NULL;
+    }
 
     return (session);
 }

src/netbios_utils.c

@@ -91,6 +91,8 @@ char  *netbios_name_encode(const char *name, char *domain,
         return (0);
 
     encoded_name = malloc(encoded_size);
+    if (!encoded_name)
+        return (0);
     encoded_name[0] = 32; // length of the field;
     netbios_name_level1_encode(name, encoded_name + 1, type);
     encoded_name[33] = 0;

src/smb_buffer.c

@@ -35,9 +35,8 @@ void    smb_buffer_alloc(smb_buffer *buf, size_t size)
     assert(buf != NULL);
 
     buf->data = malloc(size);
-    buf->size = size;
-
-    assert(buf->data != NULL);
+    if (buf->data)
+        buf->size = size;
 }
 
 void    smb_buffer_free(smb_buffer *buf)

src/smb_file.c

@@ -82,7 +82,8 @@ smb_fd      smb_fopen(smb_session *s, smb_tid tid, const char *path,
 
     resp = (smb_create_resp *)resp_msg.packet->payload;
     file = calloc(1, sizeof(smb_file));
-    assert(file != NULL);
+    if (!file)
+        return (0);
 
     file->fid           = resp->fid;
     file->tid           = tid;

src/smb_message.c

@@ -34,10 +34,14 @@ smb_message   *smb_message_new(uint8_t cmd, size_t payload_size)
     smb_message *msg;
 
     msg = (smb_message *)calloc(1, sizeof(smb_message));
-    assert(msg != NULL);
+    if (!msg)
+        return NULL;
 
     msg->packet = (smb_packet *)calloc(1, sizeof(smb_packet) + payload_size);
-    assert(msg != NULL);
+    if (!msg->packet) {
+        free(msg);
+        return NULL;
+    }
 
     msg->payload_size = payload_size;
     msg->cursor = 0;
@@ -58,12 +62,16 @@ smb_message   *smb_message_grow(smb_message *msg, size_t size)
     assert(msg != NULL && msg->packet != NULL);
 
     copy = malloc(sizeof(smb_message));
-    assert(copy != NULL);
+    if (!copy)
+        return NULL;
     copy->cursor        = msg->cursor;
     copy->payload_size  = msg->payload_size + size;
 
     copy->packet = malloc(sizeof(smb_packet) + copy->payload_size);
-    assert(copy->packet != NULL);
+    if (!copy->packet) {
+        free(copy);
+        return NULL;
+    }
     memcpy((void *)copy->packet, (void *)msg->packet,
            msg->payload_size + sizeof(smb_packet));
 

src/smb_ntlm.c

@@ -136,7 +136,10 @@ uint8_t     *smb_ntlm2_response(smb_ntlmh *hash_v2, uint64_t srv_challenge,
     HMAC_MD5(hash_v2, SMB_NTLM_HASH_SIZE, data.data, data.size, &hmac);
 
     response = malloc(blob->size + 16);
-    assert(response != NULL);
+    if (!response) {
+        smb_buffer_free(&data);
+        return NULL;
+    }
     memcpy(response, (void *)hmac, 16);
     memcpy(response + 16, blob->data, blob->size);
 
@@ -234,6 +237,8 @@ size_t      smb_ntlm_make_blob(smb_ntlm_blob **out_blob, uint64_t ts,
     assert(out_blob != NULL && target != NULL);
 
     blob = malloc(target->size + sizeof(smb_ntlm_blob));
+    if (!blob)
+        return (0);
 
     memset((void *)blob, 0, sizeof(smb_ntlm_blob));
     blob->header    = 0x101;

src/smb_session.c

@@ -36,7 +36,8 @@ smb_session   *smb_session_new()
     smb_session *s;
 
     s = calloc(1, sizeof(smb_session));
-    assert(s != NULL);
+    if (!s)
+        return NULL;
 
     s->guest              = false;
 

src/smb_share.c

@@ -81,7 +81,8 @@ smb_tid         smb_tree_connect(smb_session *s, const char *name)
 
     resp  = (smb_tree_connect_resp *)resp_msg.packet->payload;
     share = calloc(1, sizeof(smb_share));
-    assert(share != NULL);
+    if (!share)
+        return (0);
 
     share->tid          = resp_msg.packet->header.tid;
     share->opts         = resp->opt_support;
@@ -116,6 +117,8 @@ static size_t   smb_share_parse_enum(smb_message *msg, char ***list)
     eod         = msg->packet->payload + msg->payload_size;
 
     *list       = calloc(share_count + 1, sizeof(char *));
+    if (!list)
+        return (0);
     assert(*list != NULL);
 
     for (i = 0; i < share_count && data < eod; i++)

src/smb_trans2.c

@@ -50,7 +50,8 @@ static smb_file *smb_find_parse(smb_message *msg)
     {
         // Create a smb_file and fill it
         tmp = calloc(1, sizeof(smb_file));
-        assert(tmp != NULL);
+        if (!tmp)
+            return NULL;
 
         tmp->name_len = smb_from_utf16((const char *)iter->name, iter->name_len,
                                        &tmp->name);
@@ -246,7 +247,8 @@ smb_file  *smb_fstat(smb_session *s, smb_tid tid, const char *path)
     tr2_resp  = (smb_trans2_resp *)reply.packet->payload;
     info      = (smb_tr2_path_info *)(tr2_resp->payload + 4); //+4 is padding
     file      = calloc(1, sizeof(smb_file));
-    assert(file != NULL);
+    if (!file)
+        return (NULL);
 
     file->name_len  = smb_from_utf16((const char *)info->name, info->name_len,
                                      &file->name);