linux文件系统写过程简析

linux写入磁盘过程经历VFS ->  页缓存（page cache） -> 具体的文件系统（ext2/3/4、XFS、ReiserFS等） -> Block IO ->设备驱动 -> SCSI指令（或者其他指令），总体来说linux文件写入磁盘过程比较复杂

1、VFS（虚拟文件系统）

Linux中采用了VFS的方式屏蔽了多个文件系统的差别， 当需要不同的设备或者其他文件系统时，采用挂载mount的方式访问其他设备或者其他文件系统（这里可以把文件系统理解为具体的设备）。正是因为使用了VFS，所以所有的文件系统设备使用统一的文件目录树视图访问，整个存储空间采用一个文件系统目录树来管理，屏蔽了底层多个文件系统之间的差别。当然，如果你需要把你自己编写的文件系统集成到Linux内核，采用VFS的方式进行访问，你需要采用模块加载的方式进行处理，相应的文件系统模块文件需要编入到系统目录/lib/modules/your-system-name/kernel/fs当中。当然VFS的作用远不止这些，通过VFS也进行访问设备，在Linux下所有的对象都是文件，简化了系统的访问。

1.1 正常情况下，所有的文件操作通过系统调用进入到VFS中，特殊的处理，直接操作原始设备。文件系统写入的系统调用为：

　 #include <unistd.h>

　 ssize_t  write(int fd,  const void * buffer, size_t  count);

1.2 当采用系统调用进入VFS时，接下来的处理交给VFS层。处理过程比较中要的是vfs_write、generic_file_aio_write

 ssize_t vfs_write(struct file *file, const char __user *buf, size_t count, loff_t *pos)
 {
     ssize_t ret;

     if (!(file->f_mode & FMODE_WRITE))
         return -EBADF;
     if (!file->f_op || (!file->f_op->write && !file->f_op->aio_write))
         return -EINVAL;
     if (unlikely(!access_ok(VERIFY_READ, buf, count)))
         return -EFAULT;

     ret = rw_verify_area(WRITE, file, pos, count);
     if (ret >= ) {
         count = ret;
         if (file->f_op->write)
             ret = file->f_op->write(file, buf, count, pos);
         else
             ret = do_sync_write(file, buf, count, pos);
         if (ret > ) {
             fsnotify_modify(file->f_path.dentry);
             add_wchar(current, ret);
         }
         inc_syscw(current);
     }

     return ret;
 }

 /**
  * generic_file_aio_write - write data to a file
  * @iocb:    IO state structure
  * @iov:    vector with data to write
  * @nr_segs:    number of segments in the vector
  * @pos:    position in file where to write
  *
  * This is a wrapper around __generic_file_aio_write() to be used by most
  * filesystems. It takes care of syncing the file in case of O_SYNC file
  * and acquires i_mutex as needed.
  */
 ssize_t generic_file_aio_write(struct kiocb *iocb, const struct iovec *iov,
         unsigned long nr_segs, loff_t pos)
 {
     struct file *file = iocb->ki_filp;
     struct inode *inode = file->f_mapping->host;
     ssize_t ret;

     BUG_ON(iocb->ki_pos != pos);

     mutex_lock(&inode->i_mutex);
     ret = __generic_file_aio_write(iocb, iov, nr_segs, &iocb->ki_pos);
     mutex_unlock(&inode->i_mutex);

     if (ret >  || ret == -EIOCBQUEUED) {
         ssize_t err;

         err = generic_write_sync(file, pos, ret);
         if (err <  && ret > )
             ret = err;
     }
     return ret;
 }

2、 对于VFS层也有采用page cache和非page cache两种，下面重要介绍采用page cache的处理。

在VFS中， 每个打开操作的文件对应内核都有一个address_space 数据结构， 该数据结构是用来表示系统中打开的文件，并且一个打开的文件只有一个address_space数据结构。

如下：

 struct address_space {
     struct inode        *host;        /* owner: inode, block_device */
     struct radix_tree_root    page_tree;    /* radix tree of all pages */
     spinlock_t        tree_lock;    /* and lock protecting it */
     unsigned int        i_mmap_writable;/* count VM_SHARED mappings */
     struct prio_tree_root    i_mmap;        /* tree of private and shared mappings */
     struct list_head    i_mmap_nonlinear;/*list VM_NONLINEAR mappings */
     spinlock_t        i_mmap_lock;    /* protect tree, count, list */
     unsigned int        truncate_count;    /* Cover race condition with truncate */
     unsigned long        nrpages;    /* number of total pages */
     pgoff_t            writeback_index;/* writeback starts here */
     const struct address_space_operations *a_ops;    /* methods */
     unsigned long        flags;        /* error bits/gfp mask */
     struct backing_dev_info *backing_dev_info; /* device readahead, etc */
     spinlock_t        private_lock;    /* for use by the address_space */
     struct list_head    private_list;    /* ditto */
     struct address_space    *assoc_mapping;    /* ditto */
     struct mutex        unmap_mutex;    /* to protect unmapping */
 } __attribute__((aligned(sizeof(long))));

对于文件中的文件内容缓存采用的是基数树的方式来保存的，在成员变量page_tree中，关于基数树的介绍参考[1]和[2]。 下面是关于page cache写处理的几个重要的函数

 ssize_t
 generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov,
         unsigned long nr_segs, loff_t pos, loff_t *ppos,
         size_t count, ssize_t written)
 {
     struct file *file = iocb->ki_filp;
     struct address_space *mapping = file->f_mapping;
     ssize_t status;
     struct iov_iter i;

     iov_iter_init(&i, iov, nr_segs, count, written);
     status = generic_perform_write(file, &i, pos);

     if (likely(status >= )) {
         written += status;
         *ppos = pos + status;
       }

     /*
      * If we get here for O_DIRECT writes then we must have fallen through
      * to buffered writes (block instantiation inside i_size).  So we sync
      * the file data here, to try to honour O_DIRECT expectations.
      */
     if (unlikely(file->f_flags & O_DIRECT) && written)
         status = filemap_write_and_wait_range(mapping,
                     pos, pos + written - );

     return written ? written : status;
 }

调用page cache中的write_begin 和write_end

Note： 在进行VFS系统调用写入文件过程中，可以允许在文件中的任何位置写入，这其中就包括当写入的过程中写入的起始位置不是一个block的开始位置，这时需要特殊的处理，上述的过程都在write_begin这个函数调用过程中处理完毕。

3、ext2/3/4中文件的处理。

当在page cache中进行到write_begin时，需要ext4中的ext4_write_begin处理， 如下：

 static int ext4_write_begin(struct file *file, struct address_space *mapping,
                 loff_t pos, unsigned len, unsigned flags,
                 struct page **pagep, void **fsdata)
 {
     struct inode *inode = mapping->host;
     int ret, needed_blocks;
     handle_t *handle;
     int retries = ;
     struct page *page;
     pgoff_t index;
     unsigned from, to;
         .........

     index = pos >> PAGE_CACHE_SHIFT;
     from = pos & (PAGE_CACHE_SIZE - );
     to = from + len;

 retry:
     handle = ext4_journal_start(inode, needed_blocks);
     if (IS_ERR(handle)) {
         ret = PTR_ERR(handle);
         goto out;
     }

     /* We cannot recurse into the filesystem as the transaction is already
      * started */
     flags |= AOP_FLAG_NOFS;

     page = grab_cache_page_write_begin(mapping, index, flags);
     if (!page) {
         ext4_journal_stop(handle);
         ret = -ENOMEM;
         goto out;
     }
     *pagep = page;

     ret = block_write_begin(file, mapping, pos, len, flags, pagep, fsdata,
                 ext4_get_block);

     if (!ret && ext4_should_journal_data(inode)) {
         ret = walk_page_buffers(handle, page_buffers(page),
                 from, to, NULL, do_journal_get_write_access);
     }

     if (ret) {
         unlock_page(page);
         page_cache_release(page);
         /*
          * block_write_begin may have instantiated a few blocks
          * outside i_size.  Trim these off again. Don't need
          * i_size_read because we hold i_mutex.
          *
          * Add inode to orphan list in case we crash before
          * truncate finishes
          */
         if (pos + len > inode->i_size && ext4_can_truncate(inode))
             ext4_orphan_add(handle, inode);

         ext4_journal_stop(handle);
         if (pos + len > inode->i_size) {
             ext4_truncate_failed_write(inode);
             /*
              * If truncate failed early the inode might
              * still be on the orphan list; we need to
              * make sure the inode is removed from the
              * orphan list in that case.
              */
             if (inode->i_nlink)
                 ext4_orphan_del(NULL, inode);
         }
     }

     if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries))
         goto retry;
 out:
     return ret;
 }

其中在ext4_write_begin中包含了很多的处理功能，包括文件物理块的分配（假设ext4中的delay allocation特性没有开启）、文件块的部分写过程的处理等。下面是在ext_write_begin函数调用过程中比较重要的几个函数。

 /*
  * block_write_begin takes care of the basic task of block allocation and
  * bringing partial write blocks uptodate first.
  *
  * If *pagep is not NULL, then block_write_begin uses the locked page
  * at *pagep rather than allocating its own. In this case, the page will
  * not be unlocked or deallocated on failure.
  */
 int block_write_begin(struct file *file, struct address_space *mapping,
             loff_t pos, unsigned len, unsigned flags,
             struct page **pagep, void **fsdata,
             get_block_t *get_block)
 {
     struct inode *inode = mapping->host;
     int status = ;
     struct page *page;
     pgoff_t index;
     unsigned start, end;
     int ownpage = ;

     index = pos >> PAGE_CACHE_SHIFT;
     start = pos & (PAGE_CACHE_SIZE - );
     end = start + len;

     page = *pagep;
     if (page == NULL) {
         ownpage = ;
         page = grab_cache_page_write_begin(mapping, index, flags);
         if (!page) {
             status = -ENOMEM;
             goto out;
         }
         *pagep = page;
     } else
         BUG_ON(!PageLocked(page));

     status = __block_prepare_write(inode, page, start, end, get_block);
     if (unlikely(status)) {
         ClearPageUptodate(page);

         if (ownpage) {
             unlock_page(page);
             page_cache_release(page);
             *pagep = NULL;

             /*
              * prepare_write() may have instantiated a few blocks
              * outside i_size.  Trim these off again. Don't need
              * i_size_read because we hold i_mutex.
              */
             if (pos + len > inode->i_size)
                 vmtruncate(inode, inode->i_size);
         }
     }

 out:
     return status;
 }

 static int __block_prepare_write(struct inode *inode, struct page *page,
         unsigned from, unsigned to, get_block_t *get_block)
 {
     unsigned block_start, block_end;
     sector_t block;
     int err = ;
     unsigned blocksize, bbits;
     struct buffer_head *bh, *head, *wait[], **wait_bh=wait;

     BUG_ON(!PageLocked(page));
     BUG_ON(from > PAGE_CACHE_SIZE);
     BUG_ON(to > PAGE_CACHE_SIZE);
     BUG_ON(from > to);

     blocksize =  << inode->i_blkbits;
     if (!page_has_buffers(page))
         create_empty_buffers(page, blocksize, );
     head = page_buffers(page);

     bbits = inode->i_blkbits;
     block = (sector_t)page->index << (PAGE_CACHE_SHIFT - bbits);

     for(bh = head, block_start = ; bh != head || !block_start;
         block++, block_start=block_end, bh = bh->b_this_page) {
         block_end = block_start + blocksize;
         if (block_end <= from || block_start >= to) {
             if (PageUptodate(page)) {
                 if (!buffer_uptodate(bh))
                     set_buffer_uptodate(bh);
             }
             continue;
         }
         if (buffer_new(bh))
             clear_buffer_new(bh);
         if (!buffer_mapped(bh)) {
             WARN_ON(bh->b_size != blocksize);
             err = get_block(inode, block, bh, );
             if (err)
                 break;
             if (buffer_new(bh)) {
                 unmap_underlying_metadata(bh->b_bdev,
                             bh->b_blocknr);
                 if (PageUptodate(page)) {
                     clear_buffer_new(bh);
                     set_buffer_uptodate(bh);
                     mark_buffer_dirty(bh);
                     continue;
                 }
                 if (block_end > to || block_start < from)
                     zero_user_segments(page,
                         to, block_end,
                         block_start, from);
                 continue;
             }
         }
         if (PageUptodate(page)) {
             if (!buffer_uptodate(bh))
                 set_buffer_uptodate(bh);
             continue;
         }
         if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
             !buffer_unwritten(bh) &&
              (block_start < from || block_end > to)) {
             ll_rw_block(READ, , &bh);
             *wait_bh++=bh;
         }
     }
     /*
      * If we issued read requests - let them complete.
      */
     while(wait_bh > wait) {
         wait_on_buffer(*--wait_bh);
         if (!buffer_uptodate(*wait_bh))
             err = -EIO;
     }
     if (unlikely(err))
         page_zero_new_buffers(page, from, to);
     return err;
 }

 /**
  * ll_rw_block: low-level access to block devices (DEPRECATED)
  * @rw: whether to %READ or %WRITE or %SWRITE or maybe %READA (readahead)
  * @nr: number of &struct buffer_heads in the array
  * @bhs: array of pointers to &struct buffer_head
  *
  * ll_rw_block() takes an array of pointers to &struct buffer_heads, and
  * requests an I/O operation on them, either a %READ or a %WRITE.  The third
  * %SWRITE is like %WRITE only we make sure that the *current* data in buffers
  * are sent to disk. The fourth %READA option is described in the documentation
  * for generic_make_request() which ll_rw_block() calls.
  *
  * This function drops any buffer that it cannot get a lock on (with the
  * BH_Lock state bit) unless SWRITE is required, any buffer that appears to be
  * clean when doing a write request, and any buffer that appears to be
  * up-to-date when doing read request.  Further it marks as clean buffers that
  * are processed for writing (the buffer cache won't assume that they are
  * actually clean until the buffer gets unlocked).
  *
  * ll_rw_block sets b_end_io to simple completion handler that marks
  * the buffer up-to-date (if approriate), unlocks the buffer and wakes
  * any waiters.
  *
  * All of the buffers must be for the same device, and must also be a
  * multiple of the current approved size for the device.
  */
 void ll_rw_block(int rw, int nr, struct buffer_head *bhs[])
 {
     int i;

     for (i = ; i < nr; i++) {
         struct buffer_head *bh = bhs[i];

         if (rw == SWRITE || rw == SWRITE_SYNC || rw == SWRITE_SYNC_PLUG)
             lock_buffer(bh);
         else if (!trylock_buffer(bh))
             continue;

         if (rw == WRITE || rw == SWRITE || rw == SWRITE_SYNC ||
             rw == SWRITE_SYNC_PLUG) {
             if (test_clear_buffer_dirty(bh)) {
                 bh->b_end_io = end_buffer_write_sync;
                 get_bh(bh);
                 if (rw == SWRITE_SYNC)
                     submit_bh(WRITE_SYNC, bh);
                 else
                     submit_bh(WRITE, bh);
                 continue;
             }
         } else {
             if (!buffer_uptodate(bh)) {
                 bh->b_end_io = end_buffer_read_sync;
                 get_bh(bh);
                 submit_bh(rw, bh);
                 continue;
             }
         }
         unlock_buffer(bh);
     }
 }

其中在ext4中块的分配过程中，管理块分配处理的函数实现在fs/ext4/balloc.c  fs/ext4/mballoc.c

4、当page cache中的数据需要刷新到disk上的时候，这时处理的过程由Block IO接管。

在进行文件page cache刷新到disk上的过程中比较重要的数据结构有如下两个buffer_head 和 bio     

 struct buffer_head {
     unsigned long b_state;        /* buffer state bitmap (see above) */
     struct buffer_head *b_this_page;/* circular list of page's buffers */
     struct page *b_page;        /* the page this bh is mapped to */

     sector_t b_blocknr;        /* start block number */
     size_t b_size;            /* size of mapping */
     char *b_data;            /* pointer to data within the page */

     struct block_device *b_bdev;
     bh_end_io_t *b_end_io;        /* I/O completion */
      void *b_private;        /* reserved for b_end_io */
     struct list_head b_assoc_buffers; /* associated with another mapping */
     struct address_space *b_assoc_map;    /* mapping this buffer is
                            associated with */
     atomic_t b_count;        /* users using this buffer_head */
 };

 /*
  * main unit of I/O for the block layer and lower layers (ie drivers and
  * stacking drivers)
  */
 struct bio {
     sector_t        bi_sector;    /* device address in 512 byte
                            sectors */
     struct bio        *bi_next;    /* request queue link */
     struct block_device    *bi_bdev;
     unsigned long        bi_flags;    /* status, command, etc */
     unsigned long        bi_rw;        /* bottom bits READ/WRITE,
                          * top bits priority
                          */

     unsigned short        bi_vcnt;    /* how many bio_vec's */
     unsigned short        bi_idx;        /* current index into bvl_vec */
     ...............

     /*
      * We can inline a number of vecs at the end of the bio, to avoid
      * double allocations for a small number of bio_vecs. This member
      * MUST obviously be kept at the very end of the bio.
      */
     struct bio_vec        bi_inline_vecs[];
 };

在Block IO层进行基本的IO request的合并和处理调度， 基本的层由elevator管理， 具体的调度算法有noop、deadline和anticipate等多种调度算法，现在默认的调度算法是deadline，当然调度算法可调，根据系统可以调成系统最有的处理。

[1] 基数树(radix tree). http://blog.csdn.net/joker0910/article/details/8250085

[2] Radix Tree. http://en.wikipedia.org/wiki/Radix_tree