7.3.427

   1 To: vim_dev@googlegroups.com
   2 Subject: Patch 7.3.427
   3 Fcc: outbox
   4 From: Bram Moolenaar <Bram@moolenaar.net>
   5 Mime-Version: 1.0
   6 Content-Type: text/plain; charset=UTF-8
   7 Content-Transfer-Encoding: 8bit
   8 ------------
   9
  10 Patch 7.3.427
  11 Problem:    readfile() can be slow with long lines.
  12 Solution:   Use realloc() instead of alloc(). (John Little)
  13 Files:      src/eval.c
  14
  15
  16 *** ../vim-7.3.426/src/eval.c   2012-01-26 14:32:26.000000000 +0100
  17 --- src/eval.c  2012-02-05 00:25:39.000000000 +0100
  18 ***************
  19 *** 14325,14346 ****
  20       typval_T  *rettv;
  21   {
  22       int               binary = FALSE;
  23       char_u    *fname;
  24       FILE      *fd;
  25 !     listitem_T        *li;
  26 ! #define FREAD_SIZE 200            /* optimized for text lines */
  27 !     char_u    buf[FREAD_SIZE];
  28 !     int               readlen;    /* size of last fread() */
  29 !     int               buflen;     /* nr of valid chars in buf[] */
  30 !     int               filtd;      /* how much in buf[] was NUL -> '\n' filtered */
  31 !     int               tolist;     /* first byte in buf[] still to be put in list */
  32 !     int               chop;       /* how many CR to chop off */
  33 !     char_u    *prev = NULL;   /* previously read bytes, if any */
  34 !     int               prevlen = 0;    /* length of "prev" if not NULL */
  35 !     char_u    *s;
  36 !     int               len;
  37 !     long      maxline = MAXLNUM;
  38 !     long      cnt = 0;
  39
  40       if (argvars[1].v_type != VAR_UNKNOWN)
  41       {
  42 --- 14325,14343 ----
  43       typval_T  *rettv;
  44   {
  45       int               binary = FALSE;
  46 +     int               failed = FALSE;
  47       char_u    *fname;
  48       FILE      *fd;
  49 !     char_u    buf[(IOSIZE/256)*256];  /* rounded to avoid odd + 1 */
  50 !     int               io_size = sizeof(buf);
  51 !     int               readlen;                /* size of last fread() */
  52 !     char_u    *prev    = NULL;        /* previously read bytes, if any */
  53 !     long      prevlen  = 0;           /* length of data in prev */
  54 !     long      prevsize = 0;           /* size of prev buffer */
  55 !     long      maxline  = MAXLNUM;
  56 !     long      cnt      = 0;
  57 !     char_u    *p;                     /* position in buf */
  58 !     char_u    *start;                 /* start of current line */
  59
  60       if (argvars[1].v_type != VAR_UNKNOWN)
  61       {
  62 ***************
  63 *** 14362,14410 ****
  64         return;
  65       }
  66
  67 -     filtd = 0;
  68       while (cnt < maxline || maxline < 0)
  69       {
  70 !       readlen = (int)fread(buf + filtd, 1, FREAD_SIZE - filtd, fd);
  71 !       buflen = filtd + readlen;
  72 !       tolist = 0;
  73 !       for ( ; filtd < buflen || readlen <= 0; ++filtd)
  74 !       {
  75 !           if (readlen <= 0 || buf[filtd] == '\n')
  76 !           {
  77 !               /* In binary mode add an empty list item when the last
  78 !                * non-empty line ends in a '\n'. */
  79 !               if (!binary && readlen == 0 && filtd == 0 && prev == NULL)
  80 !                   break;
  81
  82 !               /* Found end-of-line or end-of-file: add a text line to the
  83 !                * list. */
  84 !               chop = 0;
  85 !               if (!binary)
  86 !                   while (filtd - chop - 1 >= tolist
  87 !                                         && buf[filtd - chop - 1] == '\r')
  88 !                       ++chop;
  89 !               len = filtd - tolist - chop;
  90 !               if (prev == NULL)
  91 !                   s = vim_strnsave(buf + tolist, len);
  92                 else
  93                 {
  94 !                   s = alloc((unsigned)(prevlen + len + 1));
  95 !                   if (s != NULL)
  96                     {
  97 !                       mch_memmove(s, prev, prevlen);
  98 !                       vim_free(prev);
  99 !                       prev = NULL;
 100 !                       mch_memmove(s + prevlen, buf + tolist, len);
 101                         s[prevlen + len] = NUL;
 102                     }
 103                 }
 104 !               tolist = filtd + 1;
 105
 106 !               li = listitem_alloc();
 107 !               if (li == NULL)
 108                 {
 109                     vim_free(s);
 110                     break;
 111                 }
 112                 li->li_tv.v_type = VAR_STRING;
 113 --- 14359,14419 ----
 114         return;
 115       }
 116
 117       while (cnt < maxline || maxline < 0)
 118       {
 119 !       readlen = (int)fread(buf, 1, io_size, fd);
 120 !
 121 !       /* This for loop processes what was read, but is also entered at end
 122 !        * of file so that either:
 123 !        * - an incomplete line gets written
 124 !        * - a "binary" file gets an empty line at the end if it ends in a
 125 !        *   newline.  */
 126 !       for (p = buf, start = buf;
 127 !               p < buf + readlen || (readlen <= 0 && (prevlen > 0 || binary));
 128 !               ++p)
 129 !       {
 130 !           if (*p == '\n' || readlen <= 0)
 131 !           {
 132 !               listitem_T  *li;
 133 !               char_u      *s  = NULL;
 134 !               long_u      len = p - start;
 135
 136 !               /* Finished a line.  Remove CRs before NL. */
 137 !               if (readlen > 0 && !binary)
 138 !               {
 139 !                   while (len > 0 && start[len - 1] == '\r')
 140 !                       --len;
 141 !                   /* removal may cross back to the "prev" string */
 142 !                   if (len == 0)
 143 !                       while (prevlen > 0 && prev[prevlen - 1] == '\r')
 144 !                           --prevlen;
 145 !               }
 146 !               if (prevlen == 0)
 147 !                   s = vim_strnsave(start, len);
 148                 else
 149                 {
 150 !                   /* Change "prev" buffer to be the right size.  This way
 151 !                    * the bytes are only copied once, and very long lines are
 152 !                    * allocated only once.  */
 153 !                   if ((s = vim_realloc(prev, prevlen + len + 1)) != NULL)
 154                     {
 155 !                       mch_memmove(s + prevlen, start, len);
 156                         s[prevlen + len] = NUL;
 157 +                       prev = NULL; /* the list will own the string */
 158 +                       prevlen = prevsize = 0;
 159                     }
 160                 }
 161 !               if (s == NULL)
 162 !               {
 163 !                   do_outofmem_msg((long_u) prevlen + len + 1);
 164 !                   failed = TRUE;
 165 !                   break;
 166 !               }
 167
 168 !               if ((li = listitem_alloc()) == NULL)
 169                 {
 170                     vim_free(s);
 171 +                   failed = TRUE;
 172                     break;
 173                 }
 174                 li->li_tv.v_type = VAR_STRING;
 175 ***************
 176 *** 14412,14485 ****
 177                 li->li_tv.vval.v_string = s;
 178                 list_append(rettv->vval.v_list, li);
 179
 180 !               if (++cnt >= maxline && maxline >= 0)
 181 !                   break;
 182 !               if (readlen <= 0)
 183                     break;
 184             }
 185 !           else if (buf[filtd] == NUL)
 186 !               buf[filtd] = '\n';
 187   #ifdef FEAT_MBYTE
 188 !           else if (buf[filtd] == 0xef
 189 !                   && enc_utf8
 190 !                   && filtd + 2 < buflen
 191 !                   && !binary
 192 !                   && buf[filtd + 1] == 0xbb
 193 !                   && buf[filtd + 2] == 0xbf)
 194 !           {
 195 !               /* remove utf-8 byte order mark */
 196 !               mch_memmove(buf + filtd, buf + filtd + 3, buflen - filtd - 3);
 197 !               --filtd;
 198 !               buflen -= 3;
 199             }
 200   #endif
 201 !       }
 202 !       if (readlen <= 0)
 203 !           break;
 204
 205 !       if (tolist == 0)
 206         {
 207 !           if (buflen >= FREAD_SIZE / 2)
 208             {
 209 !               /* "buf" is full, need to move text to an allocated buffer */
 210 !               if (prev == NULL)
 211                 {
 212 !                   prev = vim_strnsave(buf, buflen);
 213 !                   prevlen = buflen;
 214                 }
 215 !               else
 216                 {
 217 !                   s = alloc((unsigned)(prevlen + buflen));
 218 !                   if (s != NULL)
 219 !                   {
 220 !                       mch_memmove(s, prev, prevlen);
 221 !                       mch_memmove(s + prevlen, buf, buflen);
 222 !                       vim_free(prev);
 223 !                       prev = s;
 224 !                       prevlen += buflen;
 225 !                   }
 226                 }
 227 !               filtd = 0;
 228             }
 229         }
 230 !       else
 231 !       {
 232 !           mch_memmove(buf, buf + tolist, buflen - tolist);
 233 !           filtd -= tolist;
 234 !       }
 235 !     }
 236
 237       /*
 238        * For a negative line count use only the lines at the end of the file,
 239        * free the rest.
 240        */
 241 !     if (maxline < 0)
 242         while (cnt > -maxline)
 243         {
 244             listitem_remove(rettv->vval.v_list, rettv->vval.v_list->lv_first);
 245             --cnt;
 246         }
 247
 248       vim_free(prev);
 249       fclose(fd);
 250   }
 251 --- 14421,14529 ----
 252                 li->li_tv.vval.v_string = s;
 253                 list_append(rettv->vval.v_list, li);
 254
 255 !               start = p + 1; /* step over newline */
 256 !               if ((++cnt >= maxline && maxline >= 0) || readlen <= 0)
 257                     break;
 258             }
 259 !           else if (*p == NUL)
 260 !               *p = '\n';
 261   #ifdef FEAT_MBYTE
 262 !           /* Check for utf8 "bom"; U+FEFF is encoded as EF BB BF.  Do this
 263 !            * when finding the BF and check the previous two bytes. */
 264 !           else if (*p == 0xbf && enc_utf8 && !binary)
 265 !           {
 266 !               /* Find the two bytes before the 0xbf.  If p is at buf, or buf
 267 !                * + 1, these may be in the "prev" string. */
 268 !               char_u back1 = p >= buf + 1 ? p[-1]
 269 !                                    : prevlen >= 1 ? prev[prevlen - 1] : NUL;
 270 !               char_u back2 = p >= buf + 2 ? p[-2]
 271 !                         : p == buf + 1 && prevlen >= 1 ? prev[prevlen - 1]
 272 !                         : prevlen >= 2 ? prev[prevlen - 2] : NUL;
 273 !
 274 !               if (back2 == 0xef && back1 == 0xbb)
 275 !               {
 276 !                   char_u *dest = p - 2;
 277 !
 278 !                   /* Usually a BOM is at the beginning of a file, and so at
 279 !                    * the beginning of a line; then we can just step over it.
 280 !                    */
 281 !                   if (start == dest)
 282 !                       start = p + 1;
 283 !                   else
 284 !                   {
 285 !                       /* have to shuffle buf to close gap */
 286 !                       int adjust_prevlen = 0;
 287 !
 288 !                       if (dest < buf)
 289 !                       {
 290 !                           adjust_prevlen = buf - dest; /* must be 1 or 2 */
 291 !                           dest = buf;
 292 !                       }
 293 !                       if (readlen > p - buf + 1)
 294 !                           mch_memmove(dest, p + 1, readlen - (p - buf) - 1);
 295 !                       readlen -= 3 - adjust_prevlen;
 296 !                       prevlen -= adjust_prevlen;
 297 !                       p = dest - 1;
 298 !                   }
 299 !               }
 300             }
 301   #endif
 302 !       } /* for */
 303
 304 !       if (failed || (cnt >= maxline && maxline >= 0) || readlen <= 0)
 305 !           break;
 306 !       if (start < p)
 307         {
 308 !           /* There's part of a line in buf, store it in "prev". */
 309 !           if (p - start + prevlen >= prevsize)
 310             {
 311 !               /* need bigger "prev" buffer */
 312 !               char_u *newprev;
 313 !
 314 !               /* A common use case is ordinary text files and "prev" gets a
 315 !                * fragment of a line, so the first allocation is made
 316 !                * small, to avoid repeatedly 'allocing' large and
 317 !                * 'reallocing' small. */
 318 !               if (prevsize == 0)
 319 !                   prevsize = p - start;
 320 !               else
 321                 {
 322 !                   long grow50pc = (prevsize * 3) / 2;
 323 !                   long growmin  = (p - start) * 2 + prevlen;
 324 !                   prevsize = grow50pc > growmin ? grow50pc : growmin;
 325                 }
 326 !               if ((newprev = vim_realloc(prev, prevsize)) == NULL)
 327                 {
 328 !                   do_outofmem_msg((long_u)prevsize);
 329 !                   failed = TRUE;
 330 !                   break;
 331                 }
 332 !               prev = newprev;
 333             }
 334 +           /* Add the line part to end of "prev". */
 335 +           mch_memmove(prev + prevlen, start, p - start);
 336 +           prevlen += p - start;
 337         }
 338 !     } /* while */
 339
 340       /*
 341        * For a negative line count use only the lines at the end of the file,
 342        * free the rest.
 343        */
 344 !     if (!failed && maxline < 0)
 345         while (cnt > -maxline)
 346         {
 347             listitem_remove(rettv->vval.v_list, rettv->vval.v_list->lv_first);
 348             --cnt;
 349         }
 350
 351 +     if (failed)
 352 +     {
 353 +       list_free(rettv->vval.v_list, TRUE);
 354 +       /* readfile doc says an empty list is returned on error */
 355 +       rettv->vval.v_list = list_alloc();
 356 +     }
 357 +
 358       vim_free(prev);
 359       fclose(fd);
 360   }
 361 *** ../vim-7.3.426/src/version.c        2012-02-04 23:34:57.000000000 +0100
 362 --- src/version.c       2012-02-05 00:38:34.000000000 +0100
 363 ***************
 364 *** 716,717 ****
 365 --- 716,719 ----
 366   {   /* Add new patch number below this line */
 367 + /**/
 368 +     427,
 369   /**/
 370
 371 --
 372 One difference between a man and a machine is that a machine is quiet
 373 when well oiled.
 374
 375  /// Bram Moolenaar -- Bram@Moolenaar.net -- http://www.Moolenaar.net   \\\
 376 ///        sponsor Vim, vote for features -- http://www.Vim.org/sponsor/ \\\
 377 \\\  an exciting new programming language -- http://www.Zimbu.org        ///
 378  \\\            help me help AIDS victims -- http://ICCF-Holland.org    ///