binary-bug-90585.patch

   1 From c2e034be79c1e4b21d1edaa31a324d200ebe76e6 Mon Sep 17 00:00:00 2001
   2 From: =?UTF-8?q?Arkadiusz=20Mi=C5=9Bkiewicz?= <arekm@maven.pl>
   3 Date: Mon, 28 May 2018 10:01:11 +0200
   4 Subject: [PATCH] Use charsetnr for detecting binary results in mytopy_string.
   5
   6 Use returned field charsetnr attribute when doing mytopy_string conversion as documented
   7 in https://dev.mysql.com/doc/refman/5.7/en/c-api-data-structures.html
   8
   9 Metadata on the other hand is always utf8 server side but server sends
  10 it as charsetnr 63 regardless of actual charset setting. Always use utf8
  11 for metadata as documented in https://dev.mysql.com/doc/refman/5.7/en/charset-metadata.html
  12 ---
  13  src/mysql_capi.c            | 27 ++++++++++++++++-----------
  14  src/mysql_capi_conversion.c |  9 +++++----
  15  2 files changed, 21 insertions(+), 15 deletions(-)
  16
  17 diff --git a/src/mysql_capi.c b/src/mysql_capi.c
  18 index c8839a8..e72bc3f 100644
  19 --- a/src/mysql_capi.c
  20 +++ b/src/mysql_capi.c
  21 @@ -206,7 +206,7 @@ fetch_fields(MYSQL_RES *result, unsigned int num_fields, MY_CHARSET_INFO *cs,
  22      PyObject *field= NULL;
  23      PyObject *decoded= NULL;
  24      MYSQL_FIELD *myfs;
  25 -    unsigned int i;
  26 +    unsigned int i, metadata_charsetnr;
  27      char *charset= python_characterset_name(cs->csname);
  28
  29      fields = PyList_New(0);
  30 @@ -220,37 +220,41 @@ fetch_fields(MYSQL_RES *result, unsigned int num_fields, MY_CHARSET_INFO *cs,
  31      myfs = mysql_fetch_fields(result);
  32      Py_END_ALLOW_THREADS
  33
  34 +    // https://dev.mysql.com/doc/refman/5.7/en/charset-metadata.html
  35 +    // assume metadata is always utf8_bin
  36 +    metadata_charsetnr = 83;
  37 +
  38      for (i = 0; i < num_fields; i++)
  39      {
  40          field = PyTuple_New(11);
  41
  42          decoded= mytopy_string(myfs[i].catalog, myfs[i].catalog_length,
  43 -                               myfs[i].flags, charset, use_unicode);
  44 +                               metadata_charsetnr, charset, use_unicode);
  45          if (NULL == decoded) return NULL; // decode error
  46          PyTuple_SET_ITEM(field, 0, decoded);
  47
  48          decoded= mytopy_string(myfs[i].db, myfs[i].db_length,
  49 -                               myfs[i].flags, charset, use_unicode);
  50 +                               metadata_charsetnr, charset, use_unicode);
  51          if (NULL == decoded) return NULL; // decode error
  52          PyTuple_SET_ITEM(field, 1, decoded);
  53
  54          decoded= mytopy_string(myfs[i].table, myfs[i].table_length,
  55 -                               myfs[i].flags, charset, use_unicode);
  56 +                               metadata_charsetnr, charset, use_unicode);
  57          if (NULL == decoded) return NULL; // decode error
  58          PyTuple_SET_ITEM(field, 2, decoded);
  59
  60          decoded= mytopy_string(myfs[i].org_table, myfs[i].org_table_length,
  61 -                               myfs[i].flags, charset, use_unicode);
  62 +                               metadata_charsetnr, charset, use_unicode);
  63          if (NULL == decoded) return NULL; // decode error
  64          PyTuple_SET_ITEM(field, 3, decoded);
  65
  66          decoded= mytopy_string(myfs[i].name, myfs[i].name_length,
  67 -                               myfs[i].flags, charset, use_unicode);
  68 +                               metadata_charsetnr, charset, use_unicode);
  69          if (NULL == decoded) return NULL; // decode error
  70          PyTuple_SET_ITEM(field, 4, decoded);
  71
  72          decoded= mytopy_string(myfs[i].org_name, myfs[i].org_name_length,
  73 -                               myfs[i].flags, charset, use_unicode);
  74 +                               metadata_charsetnr, charset, use_unicode);
  75          if (NULL == decoded) return NULL; // decode error
  76          PyTuple_SET_ITEM(field, 5, decoded);
  77
  78 @@ -2316,7 +2320,7 @@ MySQL_fetch_row(MySQL *self)
  79         unsigned long *field_lengths;
  80         unsigned int num_fields;
  81         unsigned int i;
  82 -       unsigned long field_type, field_flags;
  83 +       unsigned long field_charsetnr, field_type, field_flags;
  84         const char *charset= NULL;
  85
  86      CHECK_SESSION(self);
  87 @@ -2391,6 +2395,7 @@ MySQL_fetch_row(MySQL *self)
  88              Py_RETURN_NONE;
  89          }
  90
  91 +        field_charsetnr= PyLong_AsUnsignedLong(PyTuple_GetItem(field_info, 6));
  92          field_type= PyLong_AsUnsignedLong(PyTuple_GetItem(field_info, 8));
  93          field_flags= PyLong_AsUnsignedLong(PyTuple_GetItem(field_info, 9));
  94
  95 @@ -2424,7 +2429,7 @@ MySQL_fetch_row(MySQL *self)
  96                   field_type == MYSQL_TYPE_ENUM ||
  97                   field_type == MYSQL_TYPE_VAR_STRING)
  98          {
  99 -            value= mytopy_string(row[i], field_lengths[i], field_flags,
 100 +            value= mytopy_string(row[i], field_lengths[i], field_charsetnr,
 101                                    charset, self->use_unicode);
 102              if (!value)
 103              {
 104 @@ -2487,7 +2492,7 @@ MySQL_fetch_row(MySQL *self)
 105          }
 106          else if (field_type == MYSQL_TYPE_BLOB)
 107          {
 108 -            value= mytopy_string(row[i], field_lengths[i], field_flags,
 109 +            value= mytopy_string(row[i], field_lengths[i], field_charsetnr,
 110                                    charset, self->use_unicode);
 111              PyTuple_SET_ITEM(result_row, i, value);
 112          }
 113 @@ -2500,7 +2505,7 @@ MySQL_fetch_row(MySQL *self)
 114         else
 115         {
 116             // Do our best to convert whatever we got from MySQL to a str/bytes
 117 -            value = mytopy_string(row[i], field_lengths[i], field_flags,
 118 +            value = mytopy_string(row[i], field_lengths[i], field_charsetnr,
 119                                    charset, self->use_unicode);
 120                 PyTuple_SET_ITEM(result_row, i, value);
 121         }
 122 diff --git a/src/mysql_capi_conversion.c b/src/mysql_capi_conversion.c
 123 index ecda11e..bd80072 100644
 124 --- a/src/mysql_capi_conversion.c
 125 +++ b/src/mysql_capi_conversion.c
 126 @@ -729,19 +729,19 @@ pytomy_decimal(PyObject *obj)
 127
 128    @param    data        string to be converted
 129    @param    length      length of data
 130 -  @param    flags       field flags
 131 +  @param    charsetnr   field charsetnr
 132    @param    charset     character used for decoding
 133    @param    use_unicode return Unicode
 134
 135    @return   Converted string
 136 -    @retval PyUnicode   if not BINARY_FLAG
 137 +    @retval PyUnicode   if not binary data
 138      @retval PyBytes     Python v3 if not use_unicode
 139      @retval PyString    Python v2 if not use_unicode
 140      @retval NULL    Exception
 141   */
 142  PyObject*
 143  mytopy_string(const char *data, const unsigned long length,
 144 -              const unsigned long flags, const char *charset,
 145 +              const unsigned long charsetnr, const char *charset,
 146                unsigned int use_unicode)
 147  {
 148      if (!charset || !data) {
 149 @@ -756,7 +756,8 @@ mytopy_string(const char *data, const un
 150          return NULL;
 151      }
 152
 153 -    if (!((flags != NULL) & flags & BINARY_FLAG) && use_unicode && strcmp(charset, "binary") != 0)
 154 +    // 63 == binary: https://dev.mysql.com/doc/internals/en/charsets.html
 155 +    if (charsetnr != 63 && use_unicode && strcmp(charset, "binary") != 0)
 156      {
 157          return PyUnicode_Decode(data, length, charset, NULL);
 158      }
 159 --
 160 2.17.0
 161