docker 镜像与容器存储目录结构

目录列表及大小示例-20220314

root@dewan01:/var/lib/docker# du -sh *

88K buildkit

72K containers

884K image

60K network

199M overlay2

16K plugins

8.0K README-my.md

4.0K runtimes

4.0K swarm

4.0K tmp

4.0K trust

12M volumes

目录说明

buildkit

containers

容器本身的目录，用来存储容器信息

命令"docker ps -a"，看到的容器ID，可以在这个目录下找到

每一个以容器ID命令的目录下，有hostname,hosts,resolv.conf等文件

image

存储相关的元数据目录

# root@dewan01:/var/lib/docker/image# tree . -L 2
.
└── overlay2
    ├── distribution
    ├── imagedb         ## 元数据，image由多个layer组成
    ├── layerdb         ## 元数据
    └── repositories.json

4 directories, 1 file

# 根据image ID过滤
# root@dewan01:/var/lib/docker/image/overlay2/imagedb# cat $(find . | grep 7ae) | jq .

  "os": "linux",
  "rootfs": {
    "type": "layers",
    "diff_ids": [
      "sha256:d31505fd5050f6b96ca3268d1db58fc91ae561ddf14eaabc41d63ea2ef8c1c6d",
      "sha256:3c7bbbb1555539bc7eba21364587faa4742400f7b3f284fb565110bc29a6b3aa",
      "sha256:f6c16e205acedafad87e9aa0185d0f107512ce8ec90b33548c895ad8f56ebb5c",
      "sha256:b8eb5ee2cdc62a5a417eb838d408b6a276b21b4e53006c885a567155fc9cd6d3",
      "sha256:c9f26c8e7aa05ad2d452eb6ff296d92e8f5885f6a5bc79af1ad9a70f0dfe9593",
      "sha256:51683fb1749e7f00845ddc3a2aa54131c7c3923fae3acee38a2035d2ca296898",
      "sha256:a543bc5c4cabe5cb050ed141c0681bc65d0d65e52f68572738b6fab67b3eb2ee",
      "sha256:a2f383f3d0ca0c56af54e56ffc860b7ff41079f76e2ec5e04595c606abdf8d34",
      "sha256:e246c8249da0a808b78029f7f6c799ca7d6b2e419d87e5455d75aeb790660c9b",
      "sha256:1e95a8217ac9eb4293228a7c4e380881d2b27620fc60469951fd3a15929cf7f9",
      "sha256:6612c9404a287d14725567594c09875fbd1d657f5b06cca75fcdbd49940745d8",
      "sha256:80f3667c795571530f1ca849a0c4e242f0cb1f00f7a2cbf3bc6d1f4ac6d14d22"
    ]

# diff_ids中的12个元素，正是组成prometheus镜像的12个layerID
# 从上往下，就是从底层到顶层。
# 去 /var/lib/docker/image/overlay2/layerdb/sha256 查看，找到了底层ID

# ll d31505fd5050f6b96ca3268d1db58fc91ae561ddf14eaabc41d63ea2ef8c1c6d/
total 32
drwx------  2 root root  4096 Mar 13 16:01 ./
drwxr-xr-x 17 root root  4096 Mar 14 09:49 ../
-rw-r--r--  1 root root    64 Mar 13 16:01 cache-id
-rw-r--r--  1 root root    71 Mar 13 16:01 diff
-rw-r--r--  1 root root     7 Mar 13 16:01 size
-rw-r--r--  1 root root 12128 Mar 13 16:01 tar-split.json.gz

# find /var/lib/docker/ | grep $(cat cache-id) | wc -l
427
# cache-id 指向的就是 overlay2 中的一个目录
# /var/lib/docker/overlay2/cf807ccea9264e13b12b502f9cc03c9b17cad6c8aabc40ff685c10fed15c5535

# 第2层id需要经过计算得到：
# echo -n "sha256:d31505fd5050f6b96ca3268d1db58fc91ae561ddf14eaabc41d63ea2ef8c1c6d sha256:3c7bbbb1555539bc7eba21364587faa4742400f7b3f284fb565110bc29a6b3aa" | sha256sum -
69d16612464e540e855466474b2cadffb861569e158ffc11bd514d503af15fbd  -

# find /var/lib/docker/ | grep $(cat cache-id) | wc -l
1911

# 以此类推，就能找到所有的layerID组合。
# 当这些layer以联合挂载的方式挂载，就能得到完整的容器rootfs

# 比较 imagedb 中记录的 sha256 ids 个数，与 layerdb ids 的个数。
# /var/lib/docker/image/overlay2/imagedb/content/sha256# ls | xargs cat $1 | jq . | grep sha256 | grep -v Image | sort | uniq | wc -l
16

# /var/lib/docker/image/overlay2/layerdb/sha256# ll | wc -l
18

network

"SandboxKey": "/var/run/docker/netns/517912ab9a5f"

overlay2

镜像和容器实际存放位置

测试中，进入到容器目录下，使用tree . -L 2，可以看到一个rootfs的目录结构。

plugins

runtimes,swarm,tmp,trust

空目录

volumes

卷目录

"Volumes": {
    "/prometheus": {}
},
"WorkingDir": "/prometheus",

比较发现，./volumes/bd2d../_data/目录下的东西和容器中/prometheus 目录下的一致。

测试&收获

# docker inspect prometheus | grep "var/lib"

        "ResolvConfPath": "/var/lib/docker/containers/163ce22c4e12f34386453f1b8954fcf77f5a6795faaec854f245709790121bb3/resolv.conf",
        "HostnamePath": "/var/lib/docker/containers/163ce22c4e12f34386453f1b8954fcf77f5a6795faaec854f245709790121bb3/hostname",
        "HostsPath": "/var/lib/docker/containers/163ce22c4e12f34386453f1b8954fcf77f5a6795faaec854f245709790121bb3/hosts",
        "LogPath": "/var/lib/docker/containers/163ce22c4e12f34386453f1b8954fcf77f5a6795faaec854f245709790121bb3/163ce22c4e12f34386453f1b8954fcf77f5a6795faaec854f245709790121bb3-json.log",
                "LowerDir": "/var/lib/docker/overlay2/b444136a9a7664a93d305583cc999147c6702e193a7b898588f7f4bc2a93a947-init/diff:/var/lib/docker/overlay2/ccd34df8e276417a6a5ba68b6b70f5b4d5d8fcd5bfb979d0b2d01506513bb79d/diff:/var/lib/docker/overlay2/34d1778940a3a78ebbb033e0221acc98b9c55e303b7f632569cd5db31febf308/diff:/var/lib/docker/overlay2/b17082d07781bc9d06f2ec14e6e5e60a947d8988b24dc50943a6c66c949788ca/diff:/var/lib/docker/overlay2/9d846358dc8cf26de3371228972d924d2cb1c7e794fe2f64e544f9167b8432bb/diff:/var/lib/docker/overlay2/deb03364f7ea3b9530dbcb245c97872884c17925986e2306f6ba31539e6bebba/diff:/var/lib/docker/overlay2/2ebfb7278b829cf0a07bcec3f6c8beb214fbd6bf140daa823415053401e1cb5d/diff:/var/lib/docker/overlay2/3040efb82ca0c4c7c5741f00ece0c37717ebeaca674996a804a0a055061e162b/diff:/var/lib/docker/overlay2/f2b0436e7337aa4123e7673961df8cd4d0f0ef4a6963b239c7558dd172cfc457/diff:/var/lib/docker/overlay2/6d8ba9c1ff481f68d7eccdac1a2341c4edf5a227d032b26d8ea8bf7cd0a29553/diff:/var/lib/docker/overlay2/affc2aa7cc3e4287aba26f69ada465f7e9f641473e8fe6805adfbe14bcfb819b/diff:/var/lib/docker/overlay2/c6fbf8097dfce2ccb31fb431f519f022b8bded2b02b4f9746f82315132fa9302/diff:/var/lib/docker/overlay2/452f128cdf975af53e61e69f5cd65e4ff27e113504d602f51508a02eca873c1a/diff:/var/lib/docker/overlay2/cf807ccea9264e13b12b502f9cc03c9b17cad6c8aabc40ff685c10fed15c5535/diff",
                "MergedDir": "/var/lib/docker/overlay2/b444136a9a7664a93d305583cc999147c6702e193a7b898588f7f4bc2a93a947/merged",
                "UpperDir": "/var/lib/docker/overlay2/b444136a9a7664a93d305583cc999147c6702e193a7b898588f7f4bc2a93a947/diff",
                "WorkDir": "/var/lib/docker/overlay2/b444136a9a7664a93d305583cc999147c6702e193a7b898588f7f4bc2a93a947/work"
                "Source": "/var/lib/docker/volumes/bd2d1872bf0376a700e03e946aed117f46c0f8a4feff4c049eef937e1bbcd40c/_data",

# root@dewan01:/var/lib/docker# find . | grep testfile
./overlay2/b444136a9a7664a93d305583cc999147c6702e193a7b898588f7f4bc2a93a947/merged/home/testfile
./overlay2/b444136a9a7664a93d305583cc999147c6702e193a7b898588f7f4bc2a93a947/diff/home/testfile

# root@dewan01:/var/lib/docker# find . | grep all_file
./volumes/bd2d1872bf0376a700e03e946aed117f46c0f8a4feff4c049eef937e1bbcd40c/_data/all_files_15-10-17

# docker pull busybox
# 增加了400多个文件
# docker images | grep busy
busybox           latest    a9d583973f65   12 months ago   1.23MB

# diff all_file.0314_1 all_file.0314_2

35a37
> ./overlay2/l/TTDQNDG6742ZRSIXROI6MLKSOS
530a533,956
> ./overlay2/b5b40fd4683a5f9c3454a78d57f8243b48021bc865963010c80923ed0b3348ec
4886a5313
> ./image/overlay2/distribution/diffid-by-digest/sha256/8b3d7e226fab303defb72e117d7668955a0999b03177a98d2762f8d62df2b559
4931a5359
> ./image/overlay2/distribution/v2metadata-by-diffid/sha256/2983725f2649f8847244cbb73ff9cb0b041bd319144816dfdee904adfd18bd1f
4964a5393
> ./image/overlay2/imagedb/content/sha256/a9d583973f65a19b3bbd7a4312b4e2c27712c44c0ed8b94e9a38cc73e7565b75
4999a5429,5433
> ./image/overlay2/layerdb/sha256/2983725f2649f8847244cbb73ff9cb0b041bd319144816dfdee904adfd18bd1f