CNI——容器网络是如何打通的

## github.com/containernetworking/cni/libcni/api.go
type CNI interface {
    AddNetworkList(net *NetworkConfigList, rt *RuntimeConf) (types.Result, error)
    DelNetworkList(net *NetworkConfigList, rt *RuntimeConf) error
    AddNetwork(net *NetworkConfig, rt *RuntimeConf) (types.Result, error)
    DelNetwork(net *NetworkConfig, rt *RuntimeConf) error
}

mkdir cni
user@ubuntu-1:~$ cd cni
user@ubuntu-1:~/cni$ curl -O -L https://github.com/containernetworking/cni/releases/download/v0.4.0/cni-amd64-v0.4.0.tgz
user@ubuntu-1:~/cni$ tar -xzvf cni-amd64-v0.4.0.tgz
user@ubuntu-1:~/cni$ ls
bridge  cni-amd64-v0.4.0.tgz  cnitool  dhcp  flannel  host-local  ipvlan  loopback  macvlan  noop  ptp  tuning

cat > mybridge.conf <<"EOF"
{
    "cniVersion": "0.2.0",
    "name": "mybridge",
    "type": "bridge",
    "bridge": "cni_bridge0",
    "isGateway": true,
    "ipMasq": true,
    "ipam": {
        "type": "host-local",
        "subnet": "10.15.20.0/24",
        "routes": [
            { "dst": "0.0.0.0/0" },
            { "dst": "1.1.1.1/32", "gw":"10.15.20.1"}
        ]
    }
}
EOF

sudo CNI_COMMAND=ADD CNI_CONTAINERID=1234567890 CNI_NETNS=/var/run/netns/1234567890 CNI_IFNAME=eth12 CNI_PATH=`pwd` ./bridge < mybridge.conf

eth12     Link encap:Ethernet  HWaddr 0a:58:0a:0f:14:02
          inet addr:10.15.20.2  Bcast:0.0.0.0  Mask:255.255.255.0
          inet6 addr: fe80::d861:8ff:fe46:33ac/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:16 errors:0 dropped:0 overruns:0 frame:0
          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:1296 (1.2 KB)  TX bytes:648 (648.0 B)
 
user@ubuntu-1:~/cni$ sudo ip netns exec 1234567890 ip route
default via 10.15.20.1 dev eth12
1.1.1.1 via 10.15.20.1 dev eth12
10.15.20.0/24 dev eth12  proto kernel  scope link  src 10.15.20.2

{
    "cniVersion": "0.2.0",
    "name": "mybridge",
    "type": "bridge",
    "bridge": "cni_bridge0",
    "isGateway": true,
    "ipMasq": true,
    "ipam": {
        "type": "host-local",
        "subnet": "10.15.20.0/24",
        "routes": [
            { "dst": "0.0.0.0/0" },
            { "dst": "1.1.1.1/32", "gw":"10.15.20.1"}
        ]
    }
}

func (ds *dockerService) RunPodSandbox(ctx context.Context, r *runtimeapi.RunPodSandboxRequest) (*runtimeapi.RunPodSandboxResponse, error) {
    config := r.GetConfig()
    // Step 1: Pull the image for the sandbox.
    err := ensureSandboxImageExists(ds.client, defaultSandboxImage);
    // Step 2: Create the sandbox container.
    createConfig, err := ds.makeSandboxDockerConfig(config, image)
    createResp, err := ds.client.CreateContainer(*createConfig)
    ds.setNetworkReady(createResp.ID, false)
    defer func(e *error) {
        // Set networking ready depending on the error return of the parent function
        if *e == nil {
            ds.setNetworkReady(createResp.ID, true)
        }
    }(&err)
    // Step 3: Create Sandbox Checkpoint.
    ds.checkpointManager.CreateCheckpoint(createResp.ID, constructPodSandboxCheckpoint(config)); 
    // Step 4: Start the sandbox container. Assume kubelet's garbage collector would remove the sandbox later, if startContainer failed.
    err = ds.client.StartContainer(createResp.ID)
    // Rewrite resolv.conf file generated by docker.
    containerInfo, err := ds.client.InspectContainer(createResp.ID)
    err := rewriteResolvFile(containerInfo.ResolvConfPath, dnsConfig.Servers, dnsConfig.Searches, dnsConfig.Options);
    // Do not invoke network plugins if in hostNetwork mode.
    if config.GetLinux().GetSecurityContext().GetNamespaceOptions().GetNetwork() == runtimeapi.NamespaceMode_NODE {
        return resp, nil
    }
    // Step 5: Setup networking for the sandbox.
    // All pod networking is setup by a CNI plugin discovered at startup time. This plugin assigns the pod ip, sets up routes inside the sandbox,
    // creates interfaces etc. In theory, its jurisdiction ends with pod sandbox networking, but it might insert iptables rules or open ports
    // on the host as well, to satisfy parts of the pod spec that aren't recognized by the CNI standard yet.
    err = ds.network.SetUpPod(config.GetMetadata().Namespace, config.GetMetadata().Name, cID, config.Annotations, networkOptions)
    return resp, nil
}

func (plugin *cniNetworkPlugin) Init(host network.Host, hairpinMode kubeletconfig.HairpinMode, nonMasqueradeCIDR string, mtu int) error {
    err := plugin.platformInit()
    ...
    plugin.host = host
    plugin.syncNetworkConfig()
    return nil
}

func (plugin *cniNetworkPlugin) syncNetworkConfig() {
    network, err := getDefaultCNINetwork(plugin.confDir, plugin.binDirs)
    ...
    plugin.setDefaultNetwork(network)
}

func getDefaultCNINetwork(confDir string, binDirs []string) (*cniNetwork, error) {
    files, err := libcni.ConfFiles(confDir, []string{".conf", ".conflist", ".json"})
    sort.Strings(files)
    for _, confFile := range files {
        var confList *libcni.NetworkConfigList
        if strings.HasSuffix(confFile, ".conflist") {
            confList, err = libcni.ConfListFromFile(confFile)
            ...
        } 
        network := &cniNetwork{
            name:          confList.Name,
            NetworkConfig: confList,
            CNIConfig:     &libcni.CNIConfig{Path: binDirs},
        }
        return network, nil
    }
    return nil, fmt.Errorf("No valid networks found in %s", confDir)
}

func (e *RawExec) ExecPlugin(ctx context.Context, pluginPath string, stdinData []byte, environ []string) ([]byte, error) {
    stdout := &bytes.Buffer{}
    c := exec.CommandContext(ctx, pluginPath)
    c.Env = environ
    c.Stdin = bytes.NewBuffer(stdinData)
    c.Stdout = stdout
    c.Stderr = e.Stderr
    if err := c.Run(); err != nil {
        return nil, pluginErr(err, stdout.Bytes())
    }
    return stdout.Bytes(), nil
}

// github.com/projectcalico/cni-plugin/cmd/calico/calico.go
func main() {
	plugin.Main(VERSION)
}
// github.com/projectcalico/cni-plugin/pkg/plugin/plugin.go
func Main(version string) {
    ...
	skel.PluginMain(cmdAdd, nil, cmdDel,
		cniSpecVersion.PluginSupports("0.1.0", "0.2.0", "0.3.0", "0.3.1"),
		"Calico CNI plugin "+version)
}
func cmdAdd(args *skel.CmdArgs) error {...}
func cmdDel(args *skel.CmdArgs) error {...}

// github.com/projectcalico/cni-plugin/pkg/plugin/plugin.go
func cmdAdd(args *skel.CmdArgs) error {
    ...
    // 获取Calico的配置，节点的名称，Calico客户端，是为了生成这个Pod唯一的WEP(Workload Endpoint)名称，也即网卡的名称。
    calicoClient, err := utils.CreateClient(conf)
    ...
    if wepIDs.Orchestrator == api.OrchestratorKubernetes {
		if result, err = k8s.CmdAddK8s(ctx, args, conf, *wepIDs, calicoClient, endpoint); err != nil {
			return err
		}
    }
    ...
}
// github.com/projectcalico/cni-plugin/pkg/k8s/k8s.go
func CmdAddK8s(ctx context.Context, args *skel.CmdArgs, conf types.NetConf, ...) (*current.Result, error) {
    ...
    client, err := newK8sClient(conf, logger)
    if conf.IPAM.Type == "host-local" {...}
    if conf.Policy.PolicyType == "k8s" {...}
    ipAddrsNoIpam := annot["cni.projectcalico.org/ipAddrsNoIpam"]
	ipAddrs := annot["cni.projectcalico.org/ipAddrs"]
    ... // 构造routes, endpoint, 从pod 中获取annotation 等数据, 可以通过设置 pod annotation 来控制pod的 网络配置，比如ip地址等
    switch {
        case ipAddrs == "" && ipAddrsNoIpam == "":
            ...
        case ipAddrs != "" && ipAddrsNoIpam != "":
            ...
        case ipAddrsNoIpam != "":
            ...
        case ipAddrs != "":
            ...
            result, err = ipAddrsResult(ipAddrs, conf, args, logger)// 调用IPAM插件，获得一个IP地址
            ...
    }
    d, err := dataplane.GetDataplane(conf, logger)
    hostVethName, contVethMac, err := d.DoNetworking(args, result, desiredVethName, routes, endpoint, annot)
}
// github.com/projectcalico/cni-plugin/pkg/dataplane/dataplane.go
func (d *linuxDataplane) DoNetworking(	
    args *skel.CmdArgs,
	result *current.Result,
	desiredVethName string,
	routes []*net.IPNet,
	endpoint *api.WorkloadEndpoint,
	annotations map[string]string)(...){
        // Clean up if hostVeth exists.
        ns.WithNetNSPath(args.Netns, func(hostNS ns.NetNS) error {
            // 先是建立veth pair，一边是contVethName，将来是容器内的，另一边是hostVethName，将来是容器外的。 主机端 cali 开头，后面 11 位是容器的 id 开头
            veth := &netlink.Veth{
                LinkAttrs: netlink.LinkAttrs{
                    Name:  contVethName,
                    Flags: net.FlagUp,
                    MTU:   d.mtu,
                },
                PeerName: hostVethName,
            }
            netlink.LinkAdd(veth)
            // host veth 给容器外的网卡设置MAC地址，状态设置为UP
            hostVeth, err := netlink.LinkByName(hostVethName)
            netlink.LinkSetHardwareAddr(hostVeth, "EE:EE:EE:EE:EE:EE")
            netlink.LinkSetUp(hostVeth)
            // container veth 给容器内的网卡设置MAC地址，IP地址，路由等
            contVeth, err := netlink.LinkByName(contVethName)   
            // Fetch the MAC from the container Veth. This is needed by Calico.
            contVethMAC = contVeth.Attrs().HardwareAddr.String()
            if hasIPv4 {
                // Add a connected route to a dummy next hop so that a default route can be set,  169.254.1.1在这里
                gw := net.IPv4(169, 254, 1, 1)
                gwNet := &net.IPNet{IP: gw, Mask: net.CIDRMask(32, 32)}
                err := netlink.RouteAdd(
                    &netlink.Route{
                        LinkIndex: contVeth.Attrs().Index,
                        Scope:     netlink.SCOPE_LINK,
                        Dst:       gwNet,
                    },
                )
                for _, r := range routes {
                     ip.AddRoute(r, gw, contVeth)
                }
            }
            if hasIPv6 {...}
        }
        // Now add the IPs to the container side of the veth.
		for _, addr := range result.IPs {
			netlink.AddrAdd(contVeth, &netlink.Addr{IPNet: &addr.Address})
        }
        // Now that the everything has been successfully set up in the container, move the "host" end of the
		// veth into the host namespace. 将host端的网卡从容器移出去，配置路由
        netlink.LinkSetNsFd(hostVeth, int(hostNS.Fd()))
    }
    // Moving a veth between namespaces always leaves it in the "DOWN" state. Set it back to "UP" now that we're
	// back in the host namespace.
    hostVeth, err := netlink.LinkByName(hostVethName)
    netlink.LinkSetUp(hostVeth)
    // Now that the host side of the veth is moved, state set to UP, and configured with sysctls, we can add the routes to it in the host namespace.
    SetupRoutes(hostVeth, result)   // 配置外界到容器的路由
    ...
}