Kubernetes网络框架

// cmd/kubelet/app/server.go

-1、func UnsecuredKubeletDeps(s *options.KubeletServer) (*kubelet.KubeletDeps, error)

• ....
• 最后调用return &kubelet.KubeletDeps {

　　　　....

　　　　NetworkPlugins:　　ProbeNetworkPlugins(s.NetworkPluginDir, s.CNIConfDir, s.CNIBinDir),

　　　　....

　　}

// cmd/kubelet/app/plugins.go

// ProbeNetworkPlugins collects all compiled-in plugins

0、func ProbeNetworkPlugins(pluginDir, cniConfDir, cniBinDir string) []network.NetworkPlugin

• 创建allPlugins := []network.NetworkPlugin{}
• 若cniConfDir为""，则设置cniConfDir为pluginDir
• 最后调用allPlugins = append(allPlugins, cni.ProbeNetworkPlugins(cniConfDir, cniBinDir)...) ---> cni.ProbeNetworkPlugins()返回一个cniNetworkPlugin为实例的NetworkPlugin接口
• allPlugins = append(allPlugins, kubenet.NewPlugin(pluginDir))

// pkg/kubelet/kubelet.go

// NewMainKubelet instantiates a new Kubelet object along with the required internal modules.

// No initialization of Kubelet and its modules should happen here.

1、func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguraion, kubeDeps *KubeletDeps, standaloneMode bool) (*Kubelet, error)

• ......
• 调用mode, err := effectiveHairpinMode(componentconfig.HairpinMode(kubeCfg.HairpinMode), kubeCfg.ContainerRuntime, kubeCfg.NetworkPluginName)
• 调用plug, err := network.InitNetworkPlugin(kubeDeps.NetworkPlugins, kubeCfg.NetworkPluginName, &criNetworkHost{&networkHost{klet}, &network.NoopPortMappingGetter{}}, klet.hairpinMode, klet.nonMasqueradeCIDR, int(kubeCfg.NetworkPluginMTU))
• 设置binDir := kubeCfg.CNIBinDir，若binDir为""，则设置binDir = kubeCfg.NetworkPluginDir
• 设置pluginSettings := dockershim.NetworkPluginSettings{

　　　　HairpinMode:　　　　　　klet.hairpinMode,

　　　　NonMasqueradeCIDR:　  klet.nonMasqueradeCIDR,

　　　　PluginName:　　　　　　 kubeCfg.NetworkPluginName,

　　　　PluginConfDir:　　　　　  kubeCfg.CNIConfDir,

　　　　PluginBinDir:　　　　　　 binDir,

　　　　MTU:　　　　　　　　　　 int(kubeCfg.NetworkPluginMTU),

　　}

• 当kubeCfg.ContainerRuntime != "rkt"并且kubeCfg.EnableCRI时：
  • 设置klet.networkPlugin = nil --> kubelet defers to the runtime shim to setup networking
• 否则，当kubeCfg.ContainerRuntime为"docker"时，创建runtime := dockertools.NewDockerManager(

　　　　...

　　　　klet.networkPlugin,

　　　　// If using "kubenet", the Kubernetes network plugin that wraps CNI's bridge plugin, it knows how

　　　　// to set the hairpin veth flag so we tell the container runtime to back away from setting it.If the

　　　　// kubelet is started with any other plugin we can't sure it handles the hairpin case so we instruct

　　　　// the docker runtime to set the flag instead.

　　　　klet.hairpinMode == componentconfig.HairpinVeth && kubeCfg.NetworkPluginName != "kubenet",

　　　　...

　　)

Host, NamespaceGetter, PortMappingGetter结构如下所示：

// Host is an interface that plugins can use to access the kubelet.Plugins, other than kubenet, only require
// a way to access namespace information and port mapping information, which they can do directly through
// the embeded interfaces.
type Host interface {
　　// NamespaceGetter is a getter for sandbox information.
　　NamespaceGetter
　　// PortMappingGetter is a getter for sandbox port mapping information.
　　PortMappingGetter
　　// LegacyHost contains methods that trap back into the Kubelet. Dependence
　　// *do not* add more dependencies in this interface. In a post-cri world,
　　// network plugins will be invoked by the runtime shim, and should only
　　// require GetNetNS and GetPodPortMappings.
　　LegacyHost
}

// NamespaceGetter is an interface to retrieve namespace information for a given
// sandboxID. Typically implemented by runtime shims that are closely coupled to
// CNI plugin wrappers like kubenet.
type NamespaceGetter interface {
　　// GetNetNS returns network namespace information for the given containerID
　　GetNetNS(containerID string) (string, error)
}

// PortMappingGetter is an interface to retrieve port mapping information for a given
// sandboxID. Typically implemented by runtime shims that are closely coupled to CNI
// plugin wrappers like kubenet.
type PortMappingGetter interface {
　　// GetPodPortMappings returns sandbox port mappings information.
　　GetPodPortMappings(containerID string) ([]*hostport.PortMapping, error)
}

　　

// pkg/kubelet/network/plugins.go

// InitNetworkPlugin inits the plugin that matches networkPluginName. Plugins must have unique names.

2、func InitNetworkPlugin(plugins []NetworkPlugin, networkPluginName string, host Host, hairpinMode componentconfig.HairpinMode, nonMasqueradeCIDR string, mtu int) (NetworkPlugin, error)

• 当networkPluginName为""时，默认设置plugin := &NoopNetworkPlugin{}，再调用plug.Init(host, hairpinMode, nonMasqueradeCIDR，mtu)并返回return plug, nil
• 否则创建pluginMap := map[string]NetworkPlugin{}，遍历plugins，将plugins都插入到pluginMap中
• 创建chosenPlugin := pluginMap[networkPluginName]，若chosenPlugin不为nil，调用chosenPlugin.Init(host, hairpinMode, nonMasqueradeCIDR, mtu)

------------------------------------------------------- 以cni plugin作为例子 -------------------------------------------------------------------

// pkg/kubelet/network/cni/cni.go

func (plugin *cniNetworkPlugin) Init(host network.Host, hairpinMode componentconfig.HairpinMode, nonMasqueradeCIDR string, mtu int) error

• 调用plugin.nsenterPath, err = plugin.execer.LookPath("nsenter")
• 将plugin.host赋值为host
• 创建一个goroutine，每隔十分钟，调用plugin.syncNetworkConfig()周期性地来检测network config的更新

cniNetworkPlugin数据结构如下：

type cniNetworkPlugin struct {
　　network.NoopNetworkPlugin
　　loNetwork　　　　*cniNetwork
　　sync.RWMutex
　　defaultNetwork　　*cniNetwork
　　host　　　　　　　  network.Host
　　execer　　　　　　  utilexec.Interface
　　nsenterPath　　　　string
　　pluginDir　　　　　  string
　　binDir　　　　　　　 string
　　VendorCNIDirPrefix　string　
}

　　

// pkg/kubelet/network/cni/cni.go

func (plugin *cniNetworkPlugin) SetUpPod(namespace string, name string, id kubecontainer.ContainerID) error

• 首先调用plugin.checkInitialized()判断plugin是否初始化完成
• 调用netnsPath, err := plugin.host.GetNetNS(id.ID)获取namespace对应的net ns的路径
• 调用_, err = plugin.loNetwork.addToNetwork(name, namespace, id, netnsPath)将pod加入lo network
• 调用_, err = plugin.getDefaultNetwork().addToNetwork(name, namespace, id, netnsPath)将pod加入default network