/dev/mem同步写不能使用msync的MS_SYNC选项探究

问题

做了个测试板子的程序，里面有一项写铁电的功能，要求写入之后立即断电，重启后校验数据准确性；铁电设计是通过内存地址直接映射的，于是，使用mmap直接映射了/dev/mem文件，自然地写入之后使用msync进行同步，最后使用munmap解映射；

然而，当我运行这段程序的时候，发现msync的MS_SYNC选项进行同步的时候会返回错误，错误码是EINVAL；这就奇怪了；

查原因

1. 查看MAN手册，如下：当地址不是页的整数倍，或者参数传递错误时才返回这个结果；

 EINVAL addr  is not a multiple of PAGESIZE; or any bit other than MS_ASYNC | MS_INVALIDATE | MS_SYNC is set in flags; or both MS_SYNC
 and MS_ASYNC are set in flags.

反复验证，发现地址没问题，而且将MS_SYNC换成MS_ASYNC就没问题了，所以怀疑是内核不支持这个同步选项；为了求证，查看内核代码：

2. sys_msync这个系统调用，在校验参数时，如果不合法会返回-EINVAL，这点如上述MAN手册所描述；

 asmlinkage long sys_msync(unsigned long start, size_t len, int flags)
 {
     unsigned long end;
     struct mm_struct *mm = current->mm;
     struct vm_area_struct *vma;
     int unmapped_error = ;
     int error = -EINVAL;

     if (flags & ~(MS_ASYNC | MS_INVALIDATE | MS_SYNC))
         goto out;
     if (start & ~PAGE_MASK)
         goto out;
     if ((flags & MS_ASYNC) && (flags & MS_SYNC))
         goto out;
         ....
 }

3. 继续往下看代码，有这么一句，如果有MS_SYNC标记的话，会执行do_fsync()，出错会返回error；

 asmlinkage long sys_msync(unsigned long start, size_t len, int flags)
 {
     ...
         if ((flags & MS_SYNC) && file &&
                 (vma->vm_flags & VM_SHARED)) {
             get_file(file);
             up_read(&mm->mmap_sem);
             error = do_fsync(file, );
             fput(file);
             if (error || start >= end)
                 goto out;
             down_read(&mm->mmap_sem);
             vma = find_vma(mm, start);
         } else {
             if (start >= end) {
                 error = ;
                 goto out_unlock;
             }
             vma = vma->vm_next;
         }
     }
 out_unlock:
     up_read(&mm->mmap_sem);
 out:
     return error ? : unmapped_error;
 }

4. 在do_fsync函数中，会对file_operations和里面的fsync函数做校验，如果没有，则返回-EINVAL，基本上可以确定，正是因为该文件没有实现file_operations里面的fsync函数，所以返回参数错误了；

 long do_fsync(struct file *file, int datasync)
 {
     int ret;
     int err;
     struct address_space *mapping = file->f_mapping;

     if (!file->f_op || !file->f_op->fsync) {
         /* Why?  We can still call filemap_fdatawrite */
         ret = -EINVAL;
         goto out;
     }

     ret = filemap_fdatawrite(mapping);

     /*
      * We need to protect against concurrent writers, which could cause
      * livelocks in fsync_buffers_list().
      */
     mutex_lock(&mapping->host->i_mutex);
     err = file->f_op->fsync(file, file->f_path.dentry, datasync);
     if (!ret)
         ret = err;
     mutex_unlock(&mapping->host->i_mutex);
     err = filemap_fdatawait(mapping);
     if (!ret)
         ret = err;
 out:
     return ret;
 }

5. 我们来看看内存设备是在什么时候初始化的，如下代码，在device_create函数调用中会对一系列的内存设备进行初始化，其中包括/dev/mem；

 static int __init chr_dev_init(void)
 {
     int i;
     int err;

     err = bdi_init(&zero_bdi);
     if (err)
         return err;

     if (register_chrdev(MEM_MAJOR,"mem",&memory_fops))
         printk("unable to get major %d for memory devs\n", MEM_MAJOR);

     mem_class = class_create(THIS_MODULE, "mem");
     for (i = ; i < ARRAY_SIZE(devlist); i++)
         device_create(mem_class, NULL,
                   MKDEV(MEM_MAJOR, devlist[i].minor),
                   devlist[i].name);

     return ;
 }

6. 这个/dev/mem对应着一个操作函数，如下代码中的mem_fops：

 static const struct {
     unsigned int        minor;
     char            *name;
     umode_t            mode;
     const struct file_operations    *fops;
 } devlist[] = { /* list of minor devices */
     {, "mem",     S_IRUSR | S_IWUSR | S_IRGRP, &mem_fops},
     {, "kmem",    S_IRUSR | S_IWUSR | S_IRGRP, &kmem_fops},
     {, "null",    S_IRUGO | S_IWUGO,           &null_fops},
 #ifdef CONFIG_DEVPORT
     {, "port",    S_IRUSR | S_IWUSR | S_IRGRP, &port_fops},
 #endif
     {, "zero",    S_IRUGO | S_IWUGO,           &zero_fops},
     {, "full",    S_IRUGO | S_IWUGO,           &full_fops},
     {, "random",  S_IRUGO | S_IWUSR,           &random_fops},
     {, "urandom", S_IRUGO | S_IWUSR,           &urandom_fops},
     {,"kmsg",    S_IRUGO | S_IWUSR,           &kmsg_fops},
 #ifdef CONFIG_CRASH_DUMP
     {,"oldmem",    S_IRUSR | S_IWUSR | S_IRGRP, &oldmem_fops},
 #endif
 };

7. 看看这个mem_fops的实现，如下，可见其并没有实现fsync函数；

 static const struct file_operations mem_fops = {
     .llseek        = memory_lseek,
     .read        = read_mem,
     .write        = write_mem,
     .mmap        = mmap_mem,
     .open        = open_mem,
     .get_unmapped_area = get_unmapped_area_mem,
 };

到这，问题总算水落石出了；

8. 再来看看mmap函数的实现，里面调用了这个函数phys_mem_access_prot；

 static int mmap_mem(struct file * file, struct vm_area_struct * vma)
 {
     size_t size = vma->vm_end - vma->vm_start;

     if (!valid_mmap_phys_addr_range(vma->vm_pgoff, size))
         return -EINVAL;

     if (!private_mapping_ok(vma))
         return -ENOSYS;

     vma->vm_page_prot = phys_mem_access_prot(file, vma->vm_pgoff,
                          size,
                          vma->vm_page_prot);

     /* Remap-pfn-range will mark the range VM_IO and VM_RESERVED */
     if (remap_pfn_range(vma,
                 vma->vm_start,
                 vma->vm_pgoff,
                 size,
                 vma->vm_page_prot))
         return -EAGAIN;
     return ;
 }

9. 上面提到的这个函数，如下，其中有个是否支持不缓存的方式判断，uncached_access；

 #ifndef __HAVE_PHYS_MEM_ACCESS_PROT
 static pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
                      unsigned long size, pgprot_t vma_prot)
 {
 #ifdef pgprot_noncached
     unsigned long offset = pfn << PAGE_SHIFT;

     if (uncached_access(file, offset))
         return pgprot_noncached(vma_prot);
 #endif
     return vma_prot;
 }
 #endif

10. 进入uncached_access非缓存访问函数，可见其内部根据文件的O_SYNC选项来判断是否支持不缓存的写；

 static inline int uncached_access(struct file *file, unsigned long addr)
 {
 #if defined(__i386__) && !defined(__arch_um__)
     /*
      * On the PPro and successors, the MTRRs are used to set
      * memory types for physical addresses outside main memory,
      * so blindly setting PCD or PWT on those pages is wrong.
      * For Pentiums and earlier, the surround logic should disable
      * caching for the high addresses through the KEN pin, but
      * we maintain the tradition of paranoia in this code.
      */
     if (file->f_flags & O_SYNC)
         return ;
      return !( test_bit(X86_FEATURE_MTRR, boot_cpu_data.x86_capability) ||
           test_bit(X86_FEATURE_K6_MTRR, boot_cpu_data.x86_capability) ||
           test_bit(X86_FEATURE_CYRIX_ARR, boot_cpu_data.x86_capability) ||
           test_bit(X86_FEATURE_CENTAUR_MCR, boot_cpu_data.x86_capability) )
       && addr >= __pa(high_memory);
 #elif defined(__x86_64__) && !defined(__arch_um__)
     /*
      * This is broken because it can generate memory type aliases,
      * which can cause cache corruptions
      * But it is only available for root and we have to be bug-to-bug
      * compatible with i386.
      */
     if (file->f_flags & O_SYNC)
         return ;
     /* same behaviour as i386. PAT always set to cached and MTRRs control the
        caching behaviour.
        Hopefully a full PAT implementation will fix that soon. */
     return ;
 #elif defined(CONFIG_IA64)
     /*
      * On ia64, we ignore O_SYNC because we cannot tolerate memory attribute aliases.
      */
     return !(efi_mem_attributes(addr) & EFI_MEMORY_WB);
 #elif defined(CONFIG_MIPS)
     {
         extern int __uncached_access(struct file *file,
                          unsigned long addr);

         return __uncached_access(file, addr);
     }
 #else
     /*
      * Accessing memory above the top the kernel knows about or through a file pointer
      * that was marked O_SYNC will be done non-cached.
      */
     if (file->f_flags & O_SYNC)
         return ;
     return addr >= __pa(high_memory);
 #endif
 }

好了，分析完毕；

解决办法

在打开/dev/mem时，使用如下方式，即open增加O_SYNC选项，这个选项即上面uncached_access函数使用的判断标记，表示每次写操作都要等到数据和文件属性都同步到物理存储才返回；

 int fd = open("/dev/mem", O_RDWR|O_SYNC);

参考文章：

https://blog.csdn.net/wlp600/article/details/6893636

http://www.armadeus.org/wiki/index.php?title=FPGA_registers_access_from_Linux_userspace

https://stackoverflow.com/questions/20750176/how-to-get-writes-via-an-mmap-mapped-memory-pointer-to-flush-immediately

https://blog.csdn.net/tiantao2012/article/details/52168383?locationNum=2&fps=1