【MySQL】filesort.cc 源代码阅读笔记

最近阅读了部分MySQL排序的代码，把心得记录一下。

参考代码

MySQL:
文件： filesort.cc
函数： filesort()

排序过程伪代码

function filesort(table_sort)
{
    param.init_for_filesort();

    //是否可以使用优先队列加速？
    if (check_if_pq_applicable()) {
        // check_if_pq_applicable 已经分配了合适的sort_buffer
        // 这里初始化sort_buffer中的指针区域。sort_buffer中
        // 数据的布局参考后文。
        table_sort.init_record_pointers()
    } else {
        table_sort.alloc_sort_buffer();
    }

    //扫描，初步处理，可能已经做了分片排序或者Top-N处理（PQ）
    find_all_keys(); 

    if (all_records_is_memory) {
        //所有数据都在内存里面，直接排序
        sort_in_memory_buffer();
    } else {
        //使用了临时文件，需要多路合并排序
        merge_many_buff();
    }
}

function find_all_keys()
{
    ;

    while((row=get_next_key()) {
        //get_next_key有可能读取的是排序字段+rowid
        //也有可能读取的是排序字段+addon_fields

        if (use_pq) { //使用优先队列
            pq.push(row)
        } else {
            filesort_buffer.append(row);
            if (filesort_buffer.full()) {
                filesort_buffer.sort_buffer();
                write_to_temp_file(filesort_buffer, temp_file);
                write_temp_file_num += ;
            }
        }
    }

    ) {
        filesort_buffer.sort_buffer();
        write_to_temp_file(filesort_buffer, temp_file);
        write_temp_file_num += ;
    }
}

void Filesort_buffer::sort_buffer()
{
  if (radixsort_is_appliccable()) //是否可以用基数排序？
  {
    radixsort();
    return;
  }

  ) // 100个以内用快速排序
  {
    my_qsort2(data, count, get_ptr_compare(sort_length));
  }

  //排序的时候，直接使用内存比较
  std::stable_sort(data, n, Mem_compare(sort_length));
}

function merge_many_buff(buffs[], count)
{
//这两个数字是写死的
#define MERGEBUFF        7
#define MERGEBUFF2        15

    if (count < MERGEBUFFF2) {
        return;
    }

    //归并排序：7合1，直到剩余总分片数量不多于15个
    while (count > MERGEBUFF2) {
        int i;
        ; i <= count - MERGEBUFF *  / ; i += MERGEBUFF) {
            first = buffs + i
            last  = buffs + i + MERGEBUFF -
            merge_buffers(first, last);
        }
        first = buffs + i
        last  = buffs + count
        merge_buffers(buffs + i, buffs + count);

        count = recount_buffs()
    }
}

filesort.cc走读

首先来看下filesort的函数原型

 /**
   Sort a table.
   Creates a set of pointers that can be used to read the rows
   in sorted order. This should be done with the functions
   in records.cc.

   Before calling filesort, one must have done
   table->file->info(HA_STATUS_VARIABLE)

   The result set is stored in table->io_cache or
   table->record_pointers.

   @param      thd            Current thread
   @param      table          Table to sort
   @param      filesort       How to sort the table
   @param      sort_positions Set to TRUE if we want to force sorting by position
                              (Needed by UPDATE/INSERT or ALTER TABLE or
                               when rowids are required by executor)
   @param[out] examined_rows  Store number of examined rows here
                              This is the number of found rows before
                              applying WHERE condition.
   @param[out] found_rows     Store the number of found rows here.
                              This is the number of found rows after
                              applying WHERE condition.

   @note
     If we sort by position (like if sort_positions is 1) filesort() will
     call table->prepare_for_position().

   @retval
     HA_POS_ERROR    Error
   @retval
     \#            Number of rows in the result, could be less than
                         found_rows if LIMIT were provided.
 */

 ha_rows filesort(THD *thd, TABLE *table, Filesort *filesort,
                  bool sort_positions, ha_rows *examined_rows,
                  ha_rows *found_rows);

第一步，初始化排序基本参数

//初始化排序参数
void Sort_param::init_for_filesort(uint sortlen, TABLE *table,
                                   ulong max_length_for_sort_data,
                                   ha_rows maxrows, bool sort_positions)
{
  sort_length= sortlen;
  ref_length= table->file->ref_length;
  if (!(table->file->ha_table_flags() & HA_FAST_KEY_READ) &&
      !table->fulltext_searched && !sort_positions)
  {
    /*
      Get the descriptors of all fields whose values are appended
      to sorted fields and get its total length in addon_length.
    */
    /*
     get_addon_fields 构造查询字段的描述信息。如果返回0，表示应该使用回表排序，如果非0，用就地排序。
    */
    addon_field= get_addon_fields(max_length_for_sort_data,
                                  table->field, sort_length, &addon_length);
  }
  if (addon_field)
    res_length= addon_length;
  else
  {
    res_length= ref_length;
    /*
      The reference to the record is considered
      as an additional sorted field
    */
    /*
      这里的reference to the record 一般就是rowid, 这里会把rowid当作排序最后一个字段
    */
    sort_length+= ref_length;
  }
  rec_length= sort_length + addon_length; // 整个排序行的长度
  max_rows= maxrows;
}

第二步，判断是否可以使用优先队列

/**
  Test whether priority queue is worth using to get top elements of an
  ordered result set. If it is, then allocates buffer for required amount of
  records

  @param trace            Current trace context.
  @param param            Sort parameters.
  @param filesort_info    Filesort information.
  @param table            Table to sort.
  @param num_rows         Estimate of number of rows in source record set.
  @param memory_available Memory available for sorting.

  DESCRIPTION
    Given a query like this:
      SELECT ... FROM t ORDER BY a1,...,an LIMIT max_rows;
    This function tests whether a priority queue should be used to keep
    the result. Necessary conditions are:
    - estimate that it is actually cheaper than merge-sort
    - enough memory to store the <max_rows> records.

    If we don't have space for <max_rows> records, but we *do* have
    space for <max_rows> keys, we may rewrite 'table' to sort with
    references to records instead of additional data.
    (again, based on estimates that it will actually be cheaper).

   @retval
    true  - if it's ok to use PQ
    false - PQ will be slower than merge-sort, or there is not enough memory.
*/

bool check_if_pq_applicable(Opt_trace_context *trace,
                            Sort_param *param,
                            Filesort_info *filesort_info,
                            TABLE *table, ha_rows num_rows,
                            ulong memory_available);

第三步，全表扫描

/**
  Search after sort_keys, and write them into tempfile
  (if we run out of space in the sort_keys buffer).
  All produced sequences are guaranteed to be non-empty.

  @param param             Sorting parameter
  @param select            Use this to get source data
  @param sort_keys         Array of pointers to sort key + addon buffers.
  @param buffpek_pointers  File to write BUFFPEKs describing sorted segments
                           in tempfile.
  @param tempfile          File to write sorted sequences of sortkeys to.
  @param pq                If !NULL, use it for keeping top N elements
  @param [out] found_rows  The number of FOUND_ROWS().
                           For a query with LIMIT, this value will typically
                           be larger than the function return value.

  @note
    Basic idea:
    @verbatim
     while (get_next_sortkey())
     {
       if (using priority queue)
         push sort key into queue
       else
       {
         if (no free space in sort_keys buffers)
         {
           sort sort_keys buffer;
           dump sorted sequence to 'tempfile';
           dump BUFFPEK describing sequence location into 'buffpek_pointers';
         }
         put sort key into 'sort_keys';
       }
     }
     if (sort_keys has some elements && dumped at least once)
       sort-dump-dump as above;
     else
       don't sort, leave sort_keys array to be sorted by caller.
  @endverbatim

  @retval
    Number of records written on success.
  @retval
    HA_POS_ERROR on error.
*/

static ha_rows find_all_keys(Sort_param *param, SQL_SELECT *select,
                             Filesort_info *fs_info,
                             IO_CACHE *buffpek_pointers,
                             IO_CACHE *tempfile,
                             Bounded_queue<uchar, uchar> *pq,
                             ha_rows *found_rows)

第四步，排序

如果结果集全部在内存中，直接用Filesort_buffer::sort_buffer()。

如果需要合并排序，调用 merge_many_buffs进行合并。

 

Filesort_buffer

filesort_buffer是一块连续的内存，分为两个区域：指针区域和数据区域。指针区域靠前，排序时，只需要对指针区域排序。

指针区域由若干指针构成，每个指针存储了每个数据行的起始偏移量。在64位机器，指针长度为8字节。

数据区域由数据行构成，每个数据行分两部分。前面是排序字段，后面是其他内容（查询字段或者RowID）。

 

数据行里面存放的内容在不同模式下是不同的，排序模式的选择规则¹：

1. 如果查询字段里面有BLOB格式，使用回表模式。
2. 否则，如果排序字段加上查询字段总长度不超过max_length_for_sort_data，使用就地模式。
3. 否则，如果超过，使用回表模式。

回表模式下，数据行=排序字段+RowID，并且RowID也会参与排序²。

本地模式下，数据行=排序字段+查询字段。

/**
  Get descriptors of fields appended to sorted fields and
  calculate its total length.

  The function first finds out what fields are used in the result set.
  Then it calculates the length of the buffer to store the values of
  these fields together with the value of sort values.
  If the calculated length is not greater than max_length_for_sort_data
  the function allocates memory for an array of descriptors containing
  layouts for the values of the non-sorted fields in the buffer and
  fills them.

  @param thd                 Current thread
  @param ptabfield           Array of references to the table fields
  @param sortlength          Total length of sorted fields
  @param[out] plength        Total length of appended fields

  @note
    The null bits for the appended values are supposed to be put together
    and stored the buffer just ahead of the value of the first field.

  @return
    Pointer to the layout descriptors for the appended fields, if any
  @retval
    NULL   if we do not store field values with sort data.
*/

static SORT_ADDON_FIELD *
get_addon_fields(ulong max_length_for_sort_data,
                 Field **ptabfield, uint sortlength, uint *plength)
{
  // ...
  MY_BITMAP *read_set= (*ptabfield)->table->read_set;

  /*
    If there is a reference to a field in the query add it
    to the the set of appended fields.
    Note for future refinement:
    This this a too strong condition.
    Actually we need only the fields referred in the
    result set. And for some of them it makes sense to use
    the values directly from sorted fields.

    目前实现：只要是read_set里面的字段就作为addon_fields，有点浪费
    优化方向：只需要把结果集的字段加到addon_fields里面
    例如 select a, b from table where c>0; c 这个字段也会在 addon_fields (?)
    实际上，只需要a和b就可以
  */

  *plength= ;

  for (pfield= ptabfield; (field= *pfield) ; pfield++)
  {
    if (!bitmap_is_set(read_set, field->field_index))
      continue;
    if (field->flags & BLOB_FLAG) // 有BLOB数据时，必定使用回表排序
      ;
    length+= field->max_packed_col_length(field->pack_length());
    if (field->maybe_null())
      null_fields++;
    fields++;
  }
  if (!fields)
    ;
  length+= (null_fields+)/;

  //数据行长度超过了max_length_for_sort_data，使用回表排序
  if (length+sortlength > max_length_for_sort_data ||
      !(addonf= (SORT_ADDON_FIELD *) my_malloc(sizeof(SORT_ADDON_FIELD)*
                                               (fields+), MYF(MY_WME))))
    ;

  // 构造addon_fields ...

}

各种数据类型作为排序字段时的二进制表示

为了方便使用memcmp排序，各字段拷贝到排序字段时，都会进行特殊处理。以Decimal为例，前导空格和'+'号以及前导'0'都转换为空格字符，负号'-'转换为0x1，因为它比所有的‘数字’都要小。负数的数字部分，'0'转换为0x9, '1'转换为0x8,...依此类推。

例如 Decimal(5.2) A='12.34' B='-34.56' C='-44.56' 的二进制表示如下：
A: 0x20 0x31 0x32 0x33 0x34
B: 0x01 0x06 0x05 0x04 0x03
C: 0x01 0x04 0x05 0x04 0x03
很显然A,B,C的二进制比较和A,B,C的逻辑比较结果是一致的。

void Field_decimal::make_sort_key(uchar *to, uint length)
{
  uchar *str,*end;
  for (str=ptr,end=ptr+length;
       str != end &&
     ((my_isspace(&my_charset_bin,*str) || *str == '+' ||
       *str == ')) ;
       str++)
    *to++=' ';
  if (str == end)
    return;                    /* purecov: inspected */

  if (*str == '-')
  {
    *to++=;                    // Smaller than any number
    str++;
    while (str != end)
      if (my_isdigit(&my_charset_bin,*str))
    *to++= (' - *str++);
      else
    *to++= *str++;
  }
  else memcpy(to,str,(uint) (end-str));
}

参考资料

1. MySQL排序内部原理探秘
2. MySQL Server 5.6.34 Source Code